diff --git a/example/jsonptr/jsonptr.cc b/example/jsonptr/jsonptr.cc index 0bc1596e..e6ea7ace 100644 --- a/example/jsonptr/jsonptr.cc +++ b/example/jsonptr/jsonptr.cc @@ -245,13 +245,93 @@ parsed_token parse_next_token() { // ---- +uint8_t hex_digit(uint8_t nibble) { + nibble &= 0x0F; + if (nibble <= 9) { + return '0' + nibble; + } + return ('A' - 10) + nibble; +} + const char* handle_string(parsed_token pt) { TRY(write_dst("\"", 1)); while (true) { - TRY(write_dst(pt.data.ptr, pt.data.len)); - if ((pt.token.value_base_detail() & 1) == 0) { - break; + uint64_t vbc = pt.token.value_base_category(); + uint64_t vbd = pt.token.value_base_detail(); + + if (vbc == 3) { + TRY(write_dst(pt.data.ptr, pt.data.len)); + if ((vbd & 1) == 0) { + break; + } + + } else if (vbd < 0x0020) { + switch (vbd) { + case '\b': + TRY(write_dst("\\b", 2)); + break; + case '\f': + TRY(write_dst("\\f", 2)); + break; + case '\n': + TRY(write_dst("\\n", 2)); + break; + case '\r': + TRY(write_dst("\\r", 2)); + break; + case '\t': + TRY(write_dst("\\t", 2)); + break; + default: { + // Other bytes less than 0x0020 are valid UTF-8 but not valid in a + // JSON string. They need to remain escaped. + uint8_t esc6[6]; + esc6[0] = '\\'; + esc6[1] = 'u'; + esc6[2] = '0'; + esc6[3] = '0'; + esc6[4] = hex_digit(vbd >> 4); + esc6[5] = hex_digit(vbd >> 0); + TRY(write_dst(&esc6[0], 6)); + break; + } + } + + } else if (vbd <= 0x007F) { + switch (vbd) { + case '\"': + TRY(write_dst("\\\"", 2)); + break; + case '\\': + TRY(write_dst("\\\\", 2)); + break; + default: { + // The UTF-8 encoding takes 1 byte. + uint8_t esc0 = (uint8_t)(vbd); + TRY(write_dst(&esc0, 1)); + break; + } + } + + } else if (vbd <= 0x07FF) { + // The UTF-8 encoding takes 2 bytes. + uint8_t esc2[6]; + esc2[0] = 0xC0 | (uint8_t)((vbd >> 6)); + esc2[1] = 0x80 | (uint8_t)((vbd >> 0) & 0x3F); + TRY(write_dst(&esc2[0], 2)); + + } else if (vbd <= 0xFFFF) { + // The UTF-8 encoding takes 3 bytes. + uint8_t esc3[6]; + esc3[0] = 0xE0 | (uint8_t)((vbd >> 12)); + esc3[1] = 0x80 | (uint8_t)((vbd >> 6) & 0x3F); + esc3[2] = 0x80 | (uint8_t)((vbd >> 0) & 0x3F); + TRY(write_dst(&esc3[0], 3)); + + } else { + return "main: unexpected Unicode code point"; } + pt = parse_next_token(); if (pt.error_msg) { return pt.error_msg; diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 0b7eff05..ceed352b 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -18439,6 +18439,46 @@ const char* wuffs_json__error__internal_error_inconsistent_i_o = // ---------------- Private Consts +static const uint8_t // + wuffs_json__backslashes[256] // + WUFFS_BASE__POTENTIALLY_UNUSED = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, + 0, 0, 0, 136, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, + 141, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +static const uint8_t // + wuffs_json__hexadecimal_digits[256] // + WUFFS_BASE__POTENTIALLY_UNUSED = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 0, 0, 0, 0, 0, 0, + 0, 138, 139, 140, 141, 142, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 138, 139, 140, 141, 142, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + // ---------------- Private Initializer Prototypes // ---------------- Private Function Prototypes @@ -18545,6 +18585,10 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, uint32_t v_stack_bit = 0; uint32_t v_match = 0; uint8_t v_c = 0; + uint8_t v_backslash = 0; + uint8_t v_uni4_ok = 0; + uint32_t v_uni4_string = 0; + uint32_t v_uni4_value = 0; uint8_t v_expect = 0; uint8_t v_expect_after_value = 0; @@ -18658,12 +18702,35 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, (iop_a_src += 1, wuffs_base__make_empty_struct()); label_2_continue:; while (true) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) { + status = + wuffs_base__make_status(wuffs_base__suspension__short_read); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(3); + goto label_2_continue; + } + if (wuffs_base__load_u8be(iop_a_src) != 92) { + goto label_2_break; + } if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { status = wuffs_base__make_status( wuffs_base__suspension__short_write); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(3); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(4); goto label_2_continue; } + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(6291457)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(0)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + goto label_2_break; + } + label_2_break:; + label_3_continue:; + while (true) { + if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { + status = wuffs_base__make_status( + wuffs_base__suspension__short_write); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(5); + goto label_3_continue; + } v_string_length = 0; while (true) { if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) { @@ -18682,12 +18749,11 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } status = wuffs_base__make_status( wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(4); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(6); v_string_length = 0; - goto label_2_continue; + goto label_3_continue; } v_c = wuffs_base__load_u8be(iop_a_src); - (iop_a_src += 1, wuffs_base__make_empty_struct()); if (v_c <= 34) { if (v_c == 34) { *iop_a_dst++ = wuffs_base__make_token( @@ -18696,7 +18762,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, (((uint64_t)(((uint64_t)(v_string_length)))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; - goto label_2_break; + goto label_3_break; } else if (v_c < 32) { if (v_string_length > 0) { *iop_a_dst++ = wuffs_base__make_token( @@ -18711,10 +18777,78 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, goto exit; } } else if (v_c == 92) { + if (v_string_length > 0) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(6291457)) + << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(((uint64_t)(v_string_length)))) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + v_string_length = 0; + if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { + goto label_3_continue; + } + } + if (((uint64_t)(io2_a_src - iop_a_src)) < 2) { + status = wuffs_base__make_status( + wuffs_base__suspension__short_read); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(7); + v_string_length = 0; + goto label_3_continue; + } + v_backslash = + ((uint8_t)((wuffs_base__load_u16le(iop_a_src) >> 8))); + v_c = wuffs_json__backslashes[v_backslash]; + if (v_c > 0) { + (iop_a_src += 2, wuffs_base__make_empty_struct()); + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)((8388608 | ((uint64_t)((v_c & 127)))))) + << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(2)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + goto label_3_continue; + } else if (v_backslash == 117) { + if (((uint64_t)(io2_a_src - iop_a_src)) < 6) { + status = wuffs_base__make_status( + wuffs_base__suspension__short_read); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(8); + v_string_length = 0; + goto label_3_continue; + } + v_uni4_string = ((uint32_t)( + (((uint64_t)(wuffs_base__load_u48le(iop_a_src))) >> + 16))); + v_uni4_value = 0; + v_uni4_ok = 128; + v_c = wuffs_json__hexadecimal_digits[( + 255 & (v_uni4_string >> 0))]; + v_uni4_ok &= v_c; + v_uni4_value |= (((uint32_t)((v_c & 15))) << 12); + v_c = wuffs_json__hexadecimal_digits[( + 255 & (v_uni4_string >> 8))]; + v_uni4_ok &= v_c; + v_uni4_value |= (((uint32_t)((v_c & 15))) << 8); + v_c = wuffs_json__hexadecimal_digits[( + 255 & (v_uni4_string >> 16))]; + v_uni4_ok &= v_c; + v_uni4_value |= (((uint32_t)((v_c & 15))) << 4); + v_c = wuffs_json__hexadecimal_digits[( + 255 & (v_uni4_string >> 24))]; + v_uni4_ok &= v_c; + v_uni4_value |= (((uint32_t)((v_c & 15))) << 0); + if (v_uni4_ok != 0) { + (iop_a_src += 6, wuffs_base__make_empty_struct()); + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)((8388608 | ((uint64_t)(v_uni4_value))))) + << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(6)) + << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + goto label_3_continue; + } + } status = wuffs_base__make_status(wuffs_json__error__bad_input); goto exit; } + (iop_a_src += 1, wuffs_base__make_empty_struct()); if (v_string_length >= 65534) { *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(6291457)) @@ -18722,29 +18856,38 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, (((uint64_t)(65535)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; - goto label_2_continue; + goto label_3_continue; } v_string_length += 1; } } - label_2_break:; - while (true) { - if (((uint64_t)(io2_a_dst - iop_a_dst)) > 0) { - *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | - (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); - goto label_3_break; - } - status = - wuffs_base__make_status(wuffs_base__suspension__short_write); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(5); - } label_3_break:; + label_4_continue:; + while (true) { + if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) { + status = + wuffs_base__make_status(wuffs_base__suspension__short_read); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(9); + goto label_4_continue; + } + if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { + status = wuffs_base__make_status( + wuffs_base__suspension__short_write); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10); + goto label_4_continue; + } + (iop_a_src += 1, wuffs_base__make_empty_struct()); + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + goto label_4_break; + } + label_4_break:; if (0 == (v_expect & 1)) { v_expect = 8; goto label_0_continue; } - goto label_4_break; + goto label_5_break; } else if (v_c == 44) { if (0 == (v_expect & 4)) { status = wuffs_base__make_status(wuffs_json__error__bad_input); @@ -18785,7 +18928,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(((uint64_t)(v_number_length)))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); - goto label_5_break; + goto label_6_break; } while (v_number_length > 0) { v_number_length -= 1; @@ -18807,16 +18950,16 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } else { status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(6); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11); while (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { status = wuffs_base__make_status( wuffs_base__suspension__short_write); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(7); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(12); } } } - label_5_break:; - goto label_4_break; + label_6_break:; + goto label_5_break; } } else if (v_c == 58) { if (0 == (v_expect & 8)) { @@ -18952,7 +19095,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, goto exit; } (iop_a_src += 4, wuffs_base__make_empty_struct()); - goto label_4_break; + goto label_5_break; } } else if (v_c == 102) { v_match = wuffs_base__io_reader__match7(iop_a_src, io2_a_src, @@ -18967,7 +19110,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, goto exit; } (iop_a_src += 5, wuffs_base__make_empty_struct()); - goto label_4_break; + goto label_5_break; } } else if (v_c == 116) { v_match = wuffs_base__io_reader__match7(iop_a_src, io2_a_src, @@ -18982,20 +19125,20 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, goto exit; } (iop_a_src += 4, wuffs_base__make_empty_struct()); - goto label_4_break; + goto label_5_break; } } if (v_match == 1) { status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(8); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(13); goto label_0_continue; } } status = wuffs_base__make_status(wuffs_json__error__bad_input); goto exit; } - label_4_break:; + label_5_break:; if (v_depth == 0) { goto label_0_break; } diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs index 8420a34d..5485a5d8 100644 --- a/std/json/decode_json.wuffs +++ b/std/json/decode_json.wuffs @@ -20,6 +20,82 @@ pub status "#unsupported recursion depth" pri status "#internal error: inconsistent I/O" +pri const backslashes array[256] base.u8 = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x00 ..= 0x07. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x08 ..= 0x0F. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x10 ..= 0x17. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x18 ..= 0x1F. + 0x00, 0x00, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20 ..= 0x27. '"'. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAF, // 0x28 ..= 0x2F. '/'. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x30 ..= 0x37. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x38 ..= 0x3F. + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x40 ..= 0x47. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x48 ..= 0x4F. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x50 ..= 0x57. + 0x00, 0x00, 0x00, 0x00, 0xDC, 0x00, 0x00, 0x00, // 0x58 ..= 0x5F. '\\'. + 0x00, 0x00, 0x88, 0x00, 0x00, 0x00, 0x8C, 0x00, // 0x60 ..= 0x67. 'b', 'f'. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0x00, // 0x68 ..= 0x6F. 'n'. + 0x00, 0x00, 0x8D, 0x00, 0x89, 0x00, 0x00, 0x00, // 0x70 ..= 0x77. 'r', 't'. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x78 ..= 0x7F. + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x80 ..= 0x87. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x88 ..= 0x8F. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x90 ..= 0x97. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x98 ..= 0x9F. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xA0 ..= 0xA7. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xA8 ..= 0xAF. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xB0 ..= 0xB7. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xB8 ..= 0xBF. + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xC0 ..= 0xC7. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xC8 ..= 0xCF. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xD0 ..= 0xD7. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xD8 ..= 0xDF. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xE0 ..= 0xE7. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xE8 ..= 0xEF. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xF0 ..= 0xF7. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xF8 ..= 0xFF. +] + +pri const hexadecimal_digits array[256] base.u8 = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x00 ..= 0x07. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x08 ..= 0x0F. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x10 ..= 0x17. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x18 ..= 0x1F. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20 ..= 0x27. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x28 ..= 0x2F. + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, // 0x30 ..= 0x37. '0'-'7'. + 0x88, 0x89, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x38 ..= 0x3F. '8'-'9'. + + 0x00, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, 0x00, // 0x40 ..= 0x47. 'A'-'F'. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x48 ..= 0x4F. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x50 ..= 0x57. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x58 ..= 0x5F. + 0x00, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, 0x00, // 0x60 ..= 0x67. 'a'-'f'. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x68 ..= 0x6F. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x70 ..= 0x77. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x78 ..= 0x7F. + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x80 ..= 0x87. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x88 ..= 0x8F. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x90 ..= 0x97. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x98 ..= 0x9F. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xA0 ..= 0xA7. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xA8 ..= 0xAF. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xB0 ..= 0xB7. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xB8 ..= 0xBF. + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xC0 ..= 0xC7. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xC8 ..= 0xCF. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xD0 ..= 0xD7. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xD8 ..= 0xDF. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xE0 ..= 0xE7. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xE8 ..= 0xEF. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xF0 ..= 0xF7. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0xF8 ..= 0xFF. +] + pub struct decoder?( )( // stack is conceptually an array of bits, implemented as an array of u32. @@ -63,6 +139,10 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { var stack_bit : base.u32[..= 31] var match : base.u32[..= 2] var c : base.u8 + var backslash : base.u8 + var uni4_ok : base.u8 + var uni4_string : base.u32 + var uni4_value : base.u32[..= 0xFFFF] // expect is a bitmask of what the next token can be: // - 0x01 EXPECT_NON_STRING_VALUE. @@ -138,7 +218,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { continue.outer } whitespace_length += 1 - } + } // while.ws // Emit whitespace. if whitespace_length > 0 { @@ -171,6 +251,27 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { length: 1) args.src.skip32_fast!(actual: 1, worst_case: 1) + // If the string contents starts with an escape code, issue + // a zero-length string token (one with value_base_category + // 3) before issuing a token with value_base_category 4. + while true { + if args.src.available() <= 0 { + yield? base."$short read" + continue + } + if args.src.peek_u8() <> 0x5C { // 0x5C is '\\'. + break + } + if args.dst.available() <= 0 { + yield? base."$short write" + continue + } + args.dst.write_fast_token!( + value: 0x60_0001, + length: 0) + break + } + while.string_loop true { if args.dst.available() <= 0 { yield? base."$short write" @@ -196,7 +297,6 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { continue.string_loop } c = args.src.peek_u8() - args.src.skip32_fast!(actual: 1, worst_case: 1) if c <= 0x22 { // 0x22 is '"'. if c == 0x22 { // 0x22 is '"'. @@ -217,11 +317,65 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { } } else if c == 0x5C { // 0x5C is '\\'. - // TODO: escapes. - return "#bad input" + if string_length > 0 { + args.dst.write_fast_token!( + value: 0x60_0001, + length: string_length as base.u64) + string_length = 0 + if args.dst.available() <= 0 { + continue.string_loop + } + } + assert args.dst.available() > 0 + if args.src.available() < 2 { + yield? base."$short read" + string_length = 0 + continue.string_loop + } + backslash = (args.src.peek_u16le() >> 8) as base.u8 + c = backslashes[backslash] + if c > 0 { + args.src.skip32_fast!(actual: 2, worst_case: 2) + args.dst.write_fast_token!( + value: 0x80_0000 | ((c & 0x7F) as base.u64), + length: 2) + continue.string_loop + + } else if backslash == 0x75 { // 0x75 is 'u'. + if args.src.available() < 6 { + yield? base."$short read" + string_length = 0 + continue.string_loop + } + uni4_string = (args.src.peek_u48le_as_u64() >> 16) as base.u32 + uni4_value = 0 + uni4_ok = 0x80 + c = hexadecimal_digits[0xFF & (uni4_string >> 0)] + uni4_ok &= c + uni4_value |= ((c & 0x0F) as base.u32) << 12 + c = hexadecimal_digits[0xFF & (uni4_string >> 8)] + uni4_ok &= c + uni4_value |= ((c & 0x0F) as base.u32) << 8 + c = hexadecimal_digits[0xFF & (uni4_string >> 16)] + uni4_ok &= c + uni4_value |= ((c & 0x0F) as base.u32) << 4 + c = hexadecimal_digits[0xFF & (uni4_string >> 24)] + uni4_ok &= c + uni4_value |= ((c & 0x0F) as base.u32) << 0 + if uni4_ok <> 0 { + args.src.skip32_fast!(actual: 6, worst_case: 6) + args.dst.write_fast_token!( + value: 0x80_0000 | (uni4_value as base.u64), + length: 6) + continue.string_loop + } + + } + return "#bad input" } + args.src.skip32_fast!(actual: 1, worst_case: 1) if string_length >= 0xFFFE { args.dst.write_fast_token!( value: 0x60_0001, @@ -231,17 +385,23 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { } string_length += 1 } - } + } // while.string_loop // The trailing '"' is filler. while true { - if args.dst.available() > 0 { - args.dst.write_fast_token!( - value: 0, - length: 1) - break + if args.src.available() <= 0 { + yield? base."$short read" + continue } - yield? base."$short write" + if args.dst.available() <= 0 { + yield? base."$short write" + continue + } + args.src.skip32_fast!(actual: 1, worst_case: 1) + args.dst.write_fast_token!( + value: 0, + length: 1) + break } // As above, expect must have contained