Have std/json decode string escapes

This commit is contained in:
Nigel Tao committed 2020-02-21 14:13:37 +11:00
1 parent 93b09674a9
commit b5461bdd5e
3 files changed
+426 -43

No files matched your search

+83 -3
View File
@@ -245,13 +245,93 @@ parsed_token parse_next_token() {
// ----
uint8_t hex_digit(uint8_t nibble) {
nibble &= 0x0F;
if (nibble <= 9) {
return '0' + nibble;
}
return ('A' - 10) + nibble;
}
const char* handle_string(parsed_token pt) {
TRY(write_dst("\"", 1));
while (true) {
TRY(write_dst(pt.data.ptr, pt.data.len));
if ((pt.token.value_base_detail() & 1) == 0) {
break;
uint64_t vbc = pt.token.value_base_category();
uint64_t vbd = pt.token.value_base_detail();
if (vbc == 3) {
TRY(write_dst(pt.data.ptr, pt.data.len));
if ((vbd & 1) == 0) {
break;
}
} else if (vbd < 0x0020) {
switch (vbd) {
case '\b':
TRY(write_dst("\\b", 2));
break;
case '\f':
TRY(write_dst("\\f", 2));
break;
case '\n':
TRY(write_dst("\\n", 2));
break;
case '\r':
TRY(write_dst("\\r", 2));
break;
case '\t':
TRY(write_dst("\\t", 2));
break;
default: {
// Other bytes less than 0x0020 are valid UTF-8 but not valid in a
// JSON string. They need to remain escaped.
uint8_t esc6[6];
esc6[0] = '\\';
esc6[1] = 'u';
esc6[2] = '0';
esc6[3] = '0';
esc6[4] = hex_digit(vbd >> 4);
esc6[5] = hex_digit(vbd >> 0);
TRY(write_dst(&esc6[0], 6));
break;
}
}
} else if (vbd <= 0x007F) {
switch (vbd) {
case '\"':
TRY(write_dst("\\\"", 2));
break;
case '\\':
TRY(write_dst("\\\\", 2));
break;
default: {
// The UTF-8 encoding takes 1 byte.
uint8_t esc0 = (uint8_t)(vbd);
TRY(write_dst(&esc0, 1));
break;
}
}
} else if (vbd <= 0x07FF) {
// The UTF-8 encoding takes 2 bytes.
uint8_t esc2[6];
esc2[0] = 0xC0 | (uint8_t)((vbd >> 6));
esc2[1] = 0x80 | (uint8_t)((vbd >> 0) & 0x3F);
TRY(write_dst(&esc2[0], 2));
} else if (vbd <= 0xFFFF) {
// The UTF-8 encoding takes 3 bytes.
uint8_t esc3[6];
esc3[0] = 0xE0 | (uint8_t)((vbd >> 12));
esc3[1] = 0x80 | (uint8_t)((vbd >> 6) & 0x3F);
esc3[2] = 0x80 | (uint8_t)((vbd >> 0) & 0x3F);
TRY(write_dst(&esc3[0], 3));
} else {
return "main: unexpected Unicode code point";
}
pt = parse_next_token();
if (pt.error_msg) {
return pt.error_msg;
+172 -29
View File
@@ -18439,6 +18439,46 @@ const char* wuffs_json__error__internal_error_inconsistent_i_o =
// ---------------- Private Consts
static const uint8_t //
wuffs_json__backslashes[256] //
WUFFS_BASE__POTENTIALLY_UNUSED = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0,
0, 0, 0, 136, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0,
141, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0,
};
static const uint8_t //
wuffs_json__hexadecimal_digits[256] //
WUFFS_BASE__POTENTIALLY_UNUSED = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 0, 0, 0, 0, 0, 0,
0, 138, 139, 140, 141, 142, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 138, 139, 140, 141, 142, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
};
// ---------------- Private Initializer Prototypes
// ---------------- Private Function Prototypes
@@ -18545,6 +18585,10 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
uint32_t v_stack_bit = 0;
uint32_t v_match = 0;
uint8_t v_c = 0;
uint8_t v_backslash = 0;
uint8_t v_uni4_ok = 0;
uint32_t v_uni4_string = 0;
uint32_t v_uni4_value = 0;
uint8_t v_expect = 0;
uint8_t v_expect_after_value = 0;
@@ -18658,12 +18702,35 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
(iop_a_src += 1, wuffs_base__make_empty_struct());
label_2_continue:;
while (true) {
if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) {
status =
wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(3);
goto label_2_continue;
}
if (wuffs_base__load_u8be(iop_a_src) != 92) {
goto label_2_break;
}
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
status = wuffs_base__make_status(
wuffs_base__suspension__short_write);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(3);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(4);
goto label_2_continue;
}
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(6291457)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label_2_break;
}
label_2_break:;
label_3_continue:;
while (true) {
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
status = wuffs_base__make_status(
wuffs_base__suspension__short_write);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(5);
goto label_3_continue;
}
v_string_length = 0;
while (true) {
if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) {
@@ -18682,12 +18749,11 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
}
status = wuffs_base__make_status(
wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(4);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(6);
v_string_length = 0;
goto label_2_continue;
goto label_3_continue;
}
v_c = wuffs_base__load_u8be(iop_a_src);
(iop_a_src += 1, wuffs_base__make_empty_struct());
if (v_c <= 34) {
if (v_c == 34) {
*iop_a_dst++ = wuffs_base__make_token(
@@ -18696,7 +18762,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
(((uint64_t)(((uint64_t)(v_string_length))))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
goto label_2_break;
goto label_3_break;
} else if (v_c < 32) {
if (v_string_length > 0) {
*iop_a_dst++ = wuffs_base__make_token(
@@ -18711,10 +18777,78 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
goto exit;
}
} else if (v_c == 92) {
if (v_string_length > 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(6291457))
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(((uint64_t)(v_string_length))))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
goto label_3_continue;
}
}
if (((uint64_t)(io2_a_src - iop_a_src)) < 2) {
status = wuffs_base__make_status(
wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(7);
v_string_length = 0;
goto label_3_continue;
}
v_backslash =
((uint8_t)((wuffs_base__load_u16le(iop_a_src) >> 8)));
v_c = wuffs_json__backslashes[v_backslash];
if (v_c > 0) {
(iop_a_src += 2, wuffs_base__make_empty_struct());
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)((8388608 | ((uint64_t)((v_c & 127))))))
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label_3_continue;
} else if (v_backslash == 117) {
if (((uint64_t)(io2_a_src - iop_a_src)) < 6) {
status = wuffs_base__make_status(
wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(8);
v_string_length = 0;
goto label_3_continue;
}
v_uni4_string = ((uint32_t)(
(((uint64_t)(wuffs_base__load_u48le(iop_a_src))) >>
16)));
v_uni4_value = 0;
v_uni4_ok = 128;
v_c = wuffs_json__hexadecimal_digits[(
255 & (v_uni4_string >> 0))];
v_uni4_ok &= v_c;
v_uni4_value |= (((uint32_t)((v_c & 15))) << 12);
v_c = wuffs_json__hexadecimal_digits[(
255 & (v_uni4_string >> 8))];
v_uni4_ok &= v_c;
v_uni4_value |= (((uint32_t)((v_c & 15))) << 8);
v_c = wuffs_json__hexadecimal_digits[(
255 & (v_uni4_string >> 16))];
v_uni4_ok &= v_c;
v_uni4_value |= (((uint32_t)((v_c & 15))) << 4);
v_c = wuffs_json__hexadecimal_digits[(
255 & (v_uni4_string >> 24))];
v_uni4_ok &= v_c;
v_uni4_value |= (((uint32_t)((v_c & 15))) << 0);
if (v_uni4_ok != 0) {
(iop_a_src += 6, wuffs_base__make_empty_struct());
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)((8388608 | ((uint64_t)(v_uni4_value)))))
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(6))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label_3_continue;
}
}
status =
wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
}
(iop_a_src += 1, wuffs_base__make_empty_struct());
if (v_string_length >= 65534) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(6291457))
@@ -18722,29 +18856,38 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
(((uint64_t)(65535))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
goto label_2_continue;
goto label_3_continue;
}
v_string_length += 1;
}
}
label_2_break:;
while (true) {
if (((uint64_t)(io2_a_dst - iop_a_dst)) > 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label_3_break;
}
status =
wuffs_base__make_status(wuffs_base__suspension__short_write);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(5);
}
label_3_break:;
label_4_continue:;
while (true) {
if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) {
status =
wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(9);
goto label_4_continue;
}
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
status = wuffs_base__make_status(
wuffs_base__suspension__short_write);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10);
goto label_4_continue;
}
(iop_a_src += 1, wuffs_base__make_empty_struct());
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label_4_break;
}
label_4_break:;
if (0 == (v_expect & 1)) {
v_expect = 8;
goto label_0_continue;
}
goto label_4_break;
goto label_5_break;
} else if (v_c == 44) {
if (0 == (v_expect & 4)) {
status = wuffs_base__make_status(wuffs_json__error__bad_input);
@@ -18785,7 +18928,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(((uint64_t)(v_number_length))))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label_5_break;
goto label_6_break;
}
while (v_number_length > 0) {
v_number_length -= 1;
@@ -18807,16 +18950,16 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
} else {
status =
wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(6);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11);
while (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
status = wuffs_base__make_status(
wuffs_base__suspension__short_write);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(7);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(12);
}
}
}
label_5_break:;
goto label_4_break;
label_6_break:;
goto label_5_break;
}
} else if (v_c == 58) {
if (0 == (v_expect & 8)) {
@@ -18952,7 +19095,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
goto exit;
}
(iop_a_src += 4, wuffs_base__make_empty_struct());
goto label_4_break;
goto label_5_break;
}
} else if (v_c == 102) {
v_match = wuffs_base__io_reader__match7(iop_a_src, io2_a_src,
@@ -18967,7 +19110,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
goto exit;
}
(iop_a_src += 5, wuffs_base__make_empty_struct());
goto label_4_break;
goto label_5_break;
}
} else if (v_c == 116) {
v_match = wuffs_base__io_reader__match7(iop_a_src, io2_a_src,
@@ -18982,20 +19125,20 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
goto exit;
}
(iop_a_src += 4, wuffs_base__make_empty_struct());
goto label_4_break;
goto label_5_break;
}
}
if (v_match == 1) {
status =
wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(8);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(13);
goto label_0_continue;
}
}
status = wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
}
label_4_break:;
label_5_break:;
if (v_depth == 0) {
goto label_0_break;
}
+171 -11
View File
@@ -20,6 +20,82 @@ pub status "#unsupported recursion depth"
pri status "#internal error: inconsistent I/O"
pri const backslashes array[256] base.u8 = [
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x00 ..= 0x07.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x08 ..= 0x0F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x10 ..= 0x17.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x18 ..= 0x1F.
0x00, 0x00, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20 ..= 0x27. '"'.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAF, // 0x28 ..= 0x2F. '/'.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x30 ..= 0x37.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x38 ..= 0x3F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x40 ..= 0x47.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x48 ..= 0x4F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x50 ..= 0x57.
0x00, 0x00, 0x00, 0x00, 0xDC, 0x00, 0x00, 0x00, // 0x58 ..= 0x5F. '\\'.
0x00, 0x00, 0x88, 0x00, 0x00, 0x00, 0x8C, 0x00, // 0x60 ..= 0x67. 'b', 'f'.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0x00, // 0x68 ..= 0x6F. 'n'.
0x00, 0x00, 0x8D, 0x00, 0x89, 0x00, 0x00, 0x00, // 0x70 ..= 0x77. 'r', 't'.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x78 ..= 0x7F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x80 ..= 0x87.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x88 ..= 0x8F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x90 ..= 0x97.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x98 ..= 0x9F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xA0 ..= 0xA7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xA8 ..= 0xAF.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xB0 ..= 0xB7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xB8 ..= 0xBF.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xC0 ..= 0xC7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xC8 ..= 0xCF.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xD0 ..= 0xD7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xD8 ..= 0xDF.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xE0 ..= 0xE7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xE8 ..= 0xEF.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xF0 ..= 0xF7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xF8 ..= 0xFF.
]
pri const hexadecimal_digits array[256] base.u8 = [
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x00 ..= 0x07.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x08 ..= 0x0F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x10 ..= 0x17.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x18 ..= 0x1F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20 ..= 0x27.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x28 ..= 0x2F.
0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, // 0x30 ..= 0x37. '0'-'7'.
0x88, 0x89, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x38 ..= 0x3F. '8'-'9'.
0x00, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, 0x00, // 0x40 ..= 0x47. 'A'-'F'.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x48 ..= 0x4F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x50 ..= 0x57.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x58 ..= 0x5F.
0x00, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, 0x00, // 0x60 ..= 0x67. 'a'-'f'.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x68 ..= 0x6F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x70 ..= 0x77.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x78 ..= 0x7F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x80 ..= 0x87.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x88 ..= 0x8F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x90 ..= 0x97.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x98 ..= 0x9F.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xA0 ..= 0xA7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xA8 ..= 0xAF.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xB0 ..= 0xB7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xB8 ..= 0xBF.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xC0 ..= 0xC7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xC8 ..= 0xCF.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xD0 ..= 0xD7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xD8 ..= 0xDF.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xE0 ..= 0xE7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xE8 ..= 0xEF.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xF0 ..= 0xF7.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xF8 ..= 0xFF.
]
pub struct decoder?(
)(
// stack is conceptually an array of bits, implemented as an array of u32.
@@ -63,6 +139,10 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
var stack_bit : base.u32[..= 31]
var match : base.u32[..= 2]
var c : base.u8
var backslash : base.u8
var uni4_ok : base.u8
var uni4_string : base.u32
var uni4_value : base.u32[..= 0xFFFF]
// expect is a bitmask of what the next token can be:
// - 0x01 EXPECT_NON_STRING_VALUE.
@@ -138,7 +218,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
continue.outer
}
whitespace_length += 1
}
} // while.ws
// Emit whitespace.
if whitespace_length > 0 {
@@ -171,6 +251,27 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
length: 1)
args.src.skip32_fast!(actual: 1, worst_case: 1)
// If the string contents starts with an escape code, issue
// a zero-length string token (one with value_base_category
// 3) before issuing a token with value_base_category 4.
while true {
if args.src.available() <= 0 {
yield? base."$short read"
continue
}
if args.src.peek_u8() <> 0x5C { // 0x5C is '\\'.
break
}
if args.dst.available() <= 0 {
yield? base."$short write"
continue
}
args.dst.write_fast_token!(
value: 0x60_0001,
length: 0)
break
}
while.string_loop true {
if args.dst.available() <= 0 {
yield? base."$short write"
@@ -196,7 +297,6 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
continue.string_loop
}
c = args.src.peek_u8()
args.src.skip32_fast!(actual: 1, worst_case: 1)
if c <= 0x22 { // 0x22 is '"'.
if c == 0x22 { // 0x22 is '"'.
@@ -217,11 +317,65 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
}
} else if c == 0x5C { // 0x5C is '\\'.
// TODO: escapes.
return "#bad input"
if string_length > 0 {
args.dst.write_fast_token!(
value: 0x60_0001,
length: string_length as base.u64)
string_length = 0
if args.dst.available() <= 0 {
continue.string_loop
}
}
assert args.dst.available() > 0
if args.src.available() < 2 {
yield? base."$short read"
string_length = 0
continue.string_loop
}
backslash = (args.src.peek_u16le() >> 8) as base.u8
c = backslashes[backslash]
if c > 0 {
args.src.skip32_fast!(actual: 2, worst_case: 2)
args.dst.write_fast_token!(
value: 0x80_0000 | ((c & 0x7F) as base.u64),
length: 2)
continue.string_loop
} else if backslash == 0x75 { // 0x75 is 'u'.
if args.src.available() < 6 {
yield? base."$short read"
string_length = 0
continue.string_loop
}
uni4_string = (args.src.peek_u48le_as_u64() >> 16) as base.u32
uni4_value = 0
uni4_ok = 0x80
c = hexadecimal_digits[0xFF & (uni4_string >> 0)]
uni4_ok &= c
uni4_value |= ((c & 0x0F) as base.u32) << 12
c = hexadecimal_digits[0xFF & (uni4_string >> 8)]
uni4_ok &= c
uni4_value |= ((c & 0x0F) as base.u32) << 8
c = hexadecimal_digits[0xFF & (uni4_string >> 16)]
uni4_ok &= c
uni4_value |= ((c & 0x0F) as base.u32) << 4
c = hexadecimal_digits[0xFF & (uni4_string >> 24)]
uni4_ok &= c
uni4_value |= ((c & 0x0F) as base.u32) << 0
if uni4_ok <> 0 {
args.src.skip32_fast!(actual: 6, worst_case: 6)
args.dst.write_fast_token!(
value: 0x80_0000 | (uni4_value as base.u64),
length: 6)
continue.string_loop
}
}
return "#bad input"
}
args.src.skip32_fast!(actual: 1, worst_case: 1)
if string_length >= 0xFFFE {
args.dst.write_fast_token!(
value: 0x60_0001,
@@ -231,17 +385,23 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
}
string_length += 1
}
}
} // while.string_loop
// The trailing '"' is filler.
while true {
if args.dst.available() > 0 {
args.dst.write_fast_token!(
value: 0,
length: 1)
break
if args.src.available() <= 0 {
yield? base."$short read"
continue
}
yield? base."$short write"
if args.dst.available() <= 0 {
yield? base."$short write"
continue
}
args.src.skip32_fast!(actual: 1, worst_case: 1)
args.dst.write_fast_token!(
value: 0,
length: 1)
break
}
// As above, expect must have contained