From 1adbc9547fa336692fcb29fa06c220f1ef0be00e Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Thu, 6 Aug 2020 23:28:07 +1000 Subject: [PATCH] Split std/json's QUIRK_ALLOW_BACKSLASH_X in two --- release/c/wuffs-unsupported-snapshot.c | 333 ++++++++++++++----------- script/print-json-token-debug-format.c | 2 +- std/json/common_consts.wuffs | 5 +- std/json/decode_json.wuffs | 146 ++++++----- std/json/decode_quirks.wuffs | 60 ++--- test/c/std/json.c | 127 ++++++---- test/data/json-quirks.tokens | Bin 960 -> 976 bytes 7 files changed, 397 insertions(+), 276 deletions(-) diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 1b57ea23..d6c078e6 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -7458,6 +7458,7 @@ extern const char wuffs_json__error__bad_c0_control_code[]; extern const char wuffs_json__error__bad_utf_8[]; extern const char wuffs_json__error__bad_backslash_escape[]; extern const char wuffs_json__error__bad_input[]; +extern const char wuffs_json__error__bad_quirk_combination[]; extern const char wuffs_json__error__unsupported_number_length[]; extern const char wuffs_json__error__unsupported_recursion_depth[]; @@ -7487,25 +7488,27 @@ extern const char wuffs_json__error__unsupported_recursion_depth[]; #define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_V 1225364487 -#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X 1225364488 +#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X_AS_BYTES 1225364488 -#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_ZERO 1225364489 +#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X_AS_CODE_POINTS 1225364489 -#define WUFFS_JSON__QUIRK_ALLOW_COMMENT_BLOCK 1225364490 +#define WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_ZERO 1225364490 -#define WUFFS_JSON__QUIRK_ALLOW_COMMENT_LINE 1225364491 +#define WUFFS_JSON__QUIRK_ALLOW_COMMENT_BLOCK 1225364491 -#define WUFFS_JSON__QUIRK_ALLOW_EXTRA_COMMA 1225364492 +#define WUFFS_JSON__QUIRK_ALLOW_COMMENT_LINE 1225364492 -#define WUFFS_JSON__QUIRK_ALLOW_INF_NAN_NUMBERS 1225364493 +#define WUFFS_JSON__QUIRK_ALLOW_EXTRA_COMMA 1225364493 -#define WUFFS_JSON__QUIRK_ALLOW_LEADING_ASCII_RECORD_SEPARATOR 1225364494 +#define WUFFS_JSON__QUIRK_ALLOW_INF_NAN_NUMBERS 1225364494 -#define WUFFS_JSON__QUIRK_ALLOW_LEADING_UNICODE_BYTE_ORDER_MARK 1225364495 +#define WUFFS_JSON__QUIRK_ALLOW_LEADING_ASCII_RECORD_SEPARATOR 1225364495 -#define WUFFS_JSON__QUIRK_ALLOW_TRAILING_NEW_LINE 1225364496 +#define WUFFS_JSON__QUIRK_ALLOW_LEADING_UNICODE_BYTE_ORDER_MARK 1225364496 -#define WUFFS_JSON__QUIRK_REPLACE_INVALID_UNICODE 1225364497 +#define WUFFS_JSON__QUIRK_ALLOW_TRAILING_NEW_LINE 1225364497 + +#define WUFFS_JSON__QUIRK_REPLACE_INVALID_UNICODE 1225364498 // ---------------- Struct Declarations @@ -7595,10 +7598,11 @@ struct wuffs_json__decoder__struct { wuffs_base__vtable vtable_for__wuffs_base__token_decoder; wuffs_base__vtable null_vtable; - bool f_quirks[18]; + bool f_quirks[19]; bool f_allow_leading_ars; bool f_allow_leading_ubom; bool f_end_of_data; + uint32_t f_string_value_minor; uint32_t p_decode_tokens[1]; uint32_t p_decode_leading[1]; @@ -25240,6 +25244,7 @@ const char wuffs_json__error__bad_c0_control_code[] = "#json: bad C0 control cod const char wuffs_json__error__bad_utf_8[] = "#json: bad UTF-8"; const char wuffs_json__error__bad_backslash_escape[] = "#json: bad backslash-escape"; const char wuffs_json__error__bad_input[] = "#json: bad input"; +const char wuffs_json__error__bad_quirk_combination[] = "#json: bad quirk combination"; const char wuffs_json__error__unsupported_number_length[] = "#json: unsupported number length"; const char wuffs_json__error__unsupported_recursion_depth[] = "#json: unsupported recursion depth"; const char wuffs_json__error__internal_error_inconsistent_i_o[] = "#json: internal error: inconsistent I/O"; @@ -25286,7 +25291,7 @@ WUFFS_JSON__LUT_BACKSLASHES[256]WUFFS_BASE__POTENTIALLY_UNUSED = { static const uint8_t WUFFS_JSON__LUT_QUIRKY_BACKSLASHES_QUIRKS[8]WUFFS_BASE__POTENTIALLY_UNUSED = { - 0, 1, 3, 4, 5, 6, 7, 9, + 0, 1, 3, 4, 5, 6, 7, 10, }; static const uint8_t @@ -25482,7 +25487,7 @@ WUFFS_JSON__LUT_HEXADECIMAL_DIGITS[256]WUFFS_BASE__POTENTIALLY_UNUSED = { #define WUFFS_JSON__QUIRKS_BASE 1225364480 -#define WUFFS_JSON__QUIRKS_COUNT 18 +#define WUFFS_JSON__QUIRKS_COUNT 19 // ---------------- Private Initializer Prototypes @@ -25624,7 +25629,7 @@ wuffs_json__decoder__set_quirk_enabled( if (a_quirk >= 1225364480) { a_quirk -= 1225364480; - if (a_quirk < 18) { + if (a_quirk < 19) { self->private_impl.f_quirks[a_quirk] = a_enabled; } } @@ -25693,6 +25698,7 @@ wuffs_json__decoder__decode_tokens( uint32_t v_multi_byte_utf8 = 0; uint32_t v_backslash_x_length = 0; uint8_t v_backslash_x_ok = 0; + uint8_t v_backslash_x_value = 0; uint32_t v_backslash_x_string = 0; uint8_t v_uni4_ok = 0; uint64_t v_uni4_string = 0; @@ -25741,7 +25747,7 @@ wuffs_json__decoder__decode_tokens( status = wuffs_base__make_status(wuffs_base__note__end_of_data); goto ok; } - if (self->private_impl.f_quirks[14] || self->private_impl.f_quirks[15]) { + if (self->private_impl.f_quirks[15] || self->private_impl.f_quirks[16]) { if (a_dst) { a_dst->meta.wi = ((size_t)(iop_a_dst - a_dst->data.ptr)); } @@ -25760,6 +25766,15 @@ wuffs_json__decoder__decode_tokens( goto suspend; } } + if (self->private_impl.f_quirks[8]) { + if (self->private_impl.f_quirks[9]) { + status = wuffs_base__make_status(wuffs_json__error__bad_quirk_combination); + goto exit; + } + self->private_impl.f_string_value_minor = 4194304; + } else { + self->private_impl.f_string_value_minor = 4194306; + } v_expect = 7858; label__outer__continue:; while (true) { @@ -25820,7 +25835,10 @@ wuffs_json__decoder__decode_tokens( } if (v_class == 1) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194579)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)((self->private_impl.f_string_value_minor | + 1 | + 16 | + 256))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); (iop_a_src += 1, wuffs_base__make_empty_struct()); @@ -25837,7 +25855,7 @@ wuffs_json__decoder__decode_tokens( if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) { if (v_string_length > 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)((self->private_impl.f_string_value_minor | 1 | 512))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -25862,7 +25880,7 @@ wuffs_json__decoder__decode_tokens( (iop_a_src += 4, wuffs_base__make_empty_struct()); if (v_string_length > 65527) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)((self->private_impl.f_string_value_minor | 1 | 512))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)((v_string_length + 4))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -25877,7 +25895,7 @@ wuffs_json__decoder__decode_tokens( (iop_a_src += 1, wuffs_base__make_empty_struct()); if (v_string_length >= 65531) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)((self->private_impl.f_string_value_minor | 1 | 512))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(65532)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -25888,7 +25906,7 @@ wuffs_json__decoder__decode_tokens( } else if (v_char == 1) { if (v_string_length != 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)((self->private_impl.f_string_value_minor | 1 | 512))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -25897,7 +25915,7 @@ wuffs_json__decoder__decode_tokens( } else if (v_char == 2) { if (v_string_length > 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)((self->private_impl.f_string_value_minor | 1 | 512))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -25973,7 +25991,7 @@ wuffs_json__decoder__decode_tokens( } else { if (((uint64_t)(io2_a_src - iop_a_src)) < 12) { if (a_src && a_src->meta.closed) { - if (self->private_impl.f_quirks[17]) { + if (self->private_impl.f_quirks[18]) { (iop_a_src += 6, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(6356989)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | @@ -26024,7 +26042,7 @@ wuffs_json__decoder__decode_tokens( goto label__string_loop_outer__continue; } } - if (self->private_impl.f_quirks[17]) { + if (self->private_impl.f_quirks[18]) { if (((uint64_t)(io2_a_src - iop_a_src)) < 6) { status = wuffs_base__make_status(wuffs_json__error__internal_error_inconsistent_i_o); goto exit; @@ -26083,7 +26101,7 @@ wuffs_json__decoder__decode_tokens( (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(10)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__string_loop_outer__continue; - } else if (self->private_impl.f_quirks[17]) { + } else if (self->private_impl.f_quirks[18]) { (iop_a_src += 10, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(6356989)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | @@ -26091,42 +26109,74 @@ wuffs_json__decoder__decode_tokens( (((uint64_t)(10)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__string_loop_outer__continue; } - } else if ((v_c == 120) && self->private_impl.f_quirks[8]) { - if (((uint64_t)(io2_a_src - iop_a_src)) < 4) { - if (a_src && a_src->meta.closed) { - status = wuffs_base__make_status(wuffs_json__error__bad_backslash_escape); - goto exit; + } else if (v_c == 120) { + if (self->private_impl.f_quirks[9]) { + if (((uint64_t)(io2_a_src - iop_a_src)) < 4) { + if (a_src && a_src->meta.closed) { + status = wuffs_base__make_status(wuffs_json__error__bad_backslash_escape); + goto exit; + } + status = wuffs_base__make_status(wuffs_base__suspension__short_read); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10); + v_string_length = 0; + v_char = 0; + goto label__string_loop_outer__continue; } - status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10); - v_string_length = 0; - v_char = 0; - goto label__string_loop_outer__continue; - } - v_backslash_x_length = 0; - while ((v_backslash_x_length <= 65531) && (((uint64_t)(io2_a_src - iop_a_src)) >= 4)) { v_backslash_x_string = wuffs_base__load_u32le__no_bounds_check(iop_a_src); v_backslash_x_ok = 128; v_c = WUFFS_JSON__LUT_HEXADECIMAL_DIGITS[(255 & (v_backslash_x_string >> 16))]; v_backslash_x_ok &= v_c; + v_backslash_x_value = ((v_c & 15) << 4); v_c = WUFFS_JSON__LUT_HEXADECIMAL_DIGITS[(255 & (v_backslash_x_string >> 24))]; v_backslash_x_ok &= v_c; + v_backslash_x_value |= (v_c & 15); if ((v_backslash_x_ok == 0) || ((v_backslash_x_string & 65535) != 30812)) { - goto label__1__break; + status = wuffs_base__make_status(wuffs_json__error__bad_backslash_escape); + goto exit; } (iop_a_src += 4, wuffs_base__make_empty_struct()); - v_backslash_x_length += 4; + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)((6291456 | ((uint32_t)(v_backslash_x_value))))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | + (((uint64_t)(4)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + goto label__string_loop_outer__continue; + } else if (self->private_impl.f_quirks[8]) { + if (((uint64_t)(io2_a_src - iop_a_src)) < 4) { + if (a_src && a_src->meta.closed) { + status = wuffs_base__make_status(wuffs_json__error__bad_backslash_escape); + goto exit; + } + status = wuffs_base__make_status(wuffs_base__suspension__short_read); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11); + v_string_length = 0; + v_char = 0; + goto label__string_loop_outer__continue; + } + v_backslash_x_length = 0; + while ((v_backslash_x_length <= 65531) && (((uint64_t)(io2_a_src - iop_a_src)) >= 4)) { + v_backslash_x_string = wuffs_base__load_u32le__no_bounds_check(iop_a_src); + v_backslash_x_ok = 128; + v_c = WUFFS_JSON__LUT_HEXADECIMAL_DIGITS[(255 & (v_backslash_x_string >> 16))]; + v_backslash_x_ok &= v_c; + v_c = WUFFS_JSON__LUT_HEXADECIMAL_DIGITS[(255 & (v_backslash_x_string >> 24))]; + v_backslash_x_ok &= v_c; + if ((v_backslash_x_ok == 0) || ((v_backslash_x_string & 65535) != 30812)) { + goto label__1__break; + } + (iop_a_src += 4, wuffs_base__make_empty_struct()); + v_backslash_x_length += 4; + } + label__1__break:; + if (v_backslash_x_length == 0) { + status = wuffs_base__make_status(wuffs_json__error__bad_backslash_escape); + goto exit; + } + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)((self->private_impl.f_string_value_minor | 1 | 2048))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | + (((uint64_t)(v_backslash_x_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + goto label__string_loop_outer__continue; } - label__1__break:; - if (v_backslash_x_length == 0) { - status = wuffs_base__make_status(wuffs_json__error__bad_backslash_escape); - goto exit; - } - *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4196355)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | - (((uint64_t)(v_backslash_x_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); - goto label__string_loop_outer__continue; } status = wuffs_base__make_status(wuffs_json__error__bad_backslash_escape); goto exit; @@ -26134,7 +26184,7 @@ wuffs_json__decoder__decode_tokens( if (((uint64_t)(io2_a_src - iop_a_src)) < 2) { if (v_string_length > 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)((self->private_impl.f_string_value_minor | 1 | 512))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -26143,52 +26193,7 @@ wuffs_json__decoder__decode_tokens( } } if (a_src && a_src->meta.closed) { - if (self->private_impl.f_quirks[17]) { - *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(6356989)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | - (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); - (iop_a_src += 1, wuffs_base__make_empty_struct()); - goto label__string_loop_outer__continue; - } - status = wuffs_base__make_status(wuffs_json__error__bad_utf_8); - goto exit; - } - status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11); - v_string_length = 0; - v_char = 0; - goto label__string_loop_outer__continue; - } - v_multi_byte_utf8 = ((uint32_t)(wuffs_base__load_u16le__no_bounds_check(iop_a_src))); - if ((v_multi_byte_utf8 & 49152) == 32768) { - v_multi_byte_utf8 = ((1984 & (v_multi_byte_utf8 << 6)) | (63 & (v_multi_byte_utf8 >> 8))); - (iop_a_src += 2, wuffs_base__make_empty_struct()); - if (v_string_length >= 65528) { - *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | - (((uint64_t)((v_string_length + 2))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); - v_string_length = 0; - goto label__string_loop_outer__continue; - } - v_string_length += 2; - goto label__string_loop_inner__continue; - } - } else if (v_char == 4) { - if (((uint64_t)(io2_a_src - iop_a_src)) < 3) { - if (v_string_length > 0) { - *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | - (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); - v_string_length = 0; - if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { - goto label__string_loop_outer__continue; - } - } - if (a_src && a_src->meta.closed) { - if (self->private_impl.f_quirks[17]) { + if (self->private_impl.f_quirks[18]) { *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(6356989)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | @@ -26205,28 +26210,26 @@ wuffs_json__decoder__decode_tokens( v_char = 0; goto label__string_loop_outer__continue; } - v_multi_byte_utf8 = ((uint32_t)(wuffs_base__load_u24le__no_bounds_check(iop_a_src))); - if ((v_multi_byte_utf8 & 12632064) == 8421376) { - v_multi_byte_utf8 = ((61440 & (v_multi_byte_utf8 << 12)) | (4032 & (v_multi_byte_utf8 >> 2)) | (63 & (v_multi_byte_utf8 >> 16))); - if ((2047 < v_multi_byte_utf8) && ((v_multi_byte_utf8 < 55296) || (57343 < v_multi_byte_utf8))) { - (iop_a_src += 3, wuffs_base__make_empty_struct()); - if (v_string_length >= 65528) { - *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | - (((uint64_t)((v_string_length + 3))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); - v_string_length = 0; - goto label__string_loop_outer__continue; - } - v_string_length += 3; - goto label__string_loop_inner__continue; + v_multi_byte_utf8 = ((uint32_t)(wuffs_base__load_u16le__no_bounds_check(iop_a_src))); + if ((v_multi_byte_utf8 & 49152) == 32768) { + v_multi_byte_utf8 = ((1984 & (v_multi_byte_utf8 << 6)) | (63 & (v_multi_byte_utf8 >> 8))); + (iop_a_src += 2, wuffs_base__make_empty_struct()); + if (v_string_length >= 65528) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)((self->private_impl.f_string_value_minor | 1 | 512))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | + (((uint64_t)((v_string_length + 2))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + v_string_length = 0; + goto label__string_loop_outer__continue; } + v_string_length += 2; + goto label__string_loop_inner__continue; } - } else if (v_char == 5) { - if (((uint64_t)(io2_a_src - iop_a_src)) < 4) { + } else if (v_char == 4) { + if (((uint64_t)(io2_a_src - iop_a_src)) < 3) { if (v_string_length > 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)((self->private_impl.f_string_value_minor | 1 | 512))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -26235,7 +26238,7 @@ wuffs_json__decoder__decode_tokens( } } if (a_src && a_src->meta.closed) { - if (self->private_impl.f_quirks[17]) { + if (self->private_impl.f_quirks[18]) { *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(6356989)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | @@ -26252,6 +26255,53 @@ wuffs_json__decoder__decode_tokens( v_char = 0; goto label__string_loop_outer__continue; } + v_multi_byte_utf8 = ((uint32_t)(wuffs_base__load_u24le__no_bounds_check(iop_a_src))); + if ((v_multi_byte_utf8 & 12632064) == 8421376) { + v_multi_byte_utf8 = ((61440 & (v_multi_byte_utf8 << 12)) | (4032 & (v_multi_byte_utf8 >> 2)) | (63 & (v_multi_byte_utf8 >> 16))); + if ((2047 < v_multi_byte_utf8) && ((v_multi_byte_utf8 < 55296) || (57343 < v_multi_byte_utf8))) { + (iop_a_src += 3, wuffs_base__make_empty_struct()); + if (v_string_length >= 65528) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)((self->private_impl.f_string_value_minor | 1 | 512))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | + (((uint64_t)((v_string_length + 3))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + v_string_length = 0; + goto label__string_loop_outer__continue; + } + v_string_length += 3; + goto label__string_loop_inner__continue; + } + } + } else if (v_char == 5) { + if (((uint64_t)(io2_a_src - iop_a_src)) < 4) { + if (v_string_length > 0) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)((self->private_impl.f_string_value_minor | 1 | 512))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | + (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + v_string_length = 0; + if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { + goto label__string_loop_outer__continue; + } + } + if (a_src && a_src->meta.closed) { + if (self->private_impl.f_quirks[18]) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(6356989)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + (iop_a_src += 1, wuffs_base__make_empty_struct()); + goto label__string_loop_outer__continue; + } + status = wuffs_base__make_status(wuffs_json__error__bad_utf_8); + goto exit; + } + status = wuffs_base__make_status(wuffs_base__suspension__short_read); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(14); + v_string_length = 0; + v_char = 0; + goto label__string_loop_outer__continue; + } v_multi_byte_utf8 = wuffs_base__load_u32le__no_bounds_check(iop_a_src); if ((v_multi_byte_utf8 & 3233857536) == 2155905024) { v_multi_byte_utf8 = ((1835008 & (v_multi_byte_utf8 << 18)) | @@ -26262,7 +26312,7 @@ wuffs_json__decoder__decode_tokens( (iop_a_src += 4, wuffs_base__make_empty_struct()); if (v_string_length >= 65528) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)((self->private_impl.f_string_value_minor | 1 | 512))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)((v_string_length + 4))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -26275,7 +26325,7 @@ wuffs_json__decoder__decode_tokens( } if (v_string_length > 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)((self->private_impl.f_string_value_minor | 1 | 512))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -26295,7 +26345,7 @@ wuffs_json__decoder__decode_tokens( status = wuffs_base__make_status(wuffs_json__error__bad_c0_control_code); goto exit; } - if (self->private_impl.f_quirks[17]) { + if (self->private_impl.f_quirks[18]) { *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(6356989)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | @@ -26316,17 +26366,20 @@ wuffs_json__decoder__decode_tokens( goto exit; } status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(14); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(15); goto label__2__continue; } if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { status = wuffs_base__make_status(wuffs_base__suspension__short_write); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(15); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(16); goto label__2__continue; } (iop_a_src += 1, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(4194579)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)((self->private_impl.f_string_value_minor | + 1 | + 16 | + 256))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__2__break; } @@ -26342,13 +26395,13 @@ wuffs_json__decoder__decode_tokens( (((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); if (0 == (v_expect & (((uint32_t)(1)) << 8))) { - if (self->private_impl.f_quirks[12]) { + if (self->private_impl.f_quirks[13]) { v_expect = 4162; } else { v_expect = 4098; } } else { - if (self->private_impl.f_quirks[12]) { + if (self->private_impl.f_quirks[13]) { v_expect = 8114; } else { v_expect = 7858; @@ -26393,14 +26446,14 @@ wuffs_json__decoder__decode_tokens( } } if (v_number_status == 1) { - if (self->private_impl.f_quirks[13]) { + if (self->private_impl.f_quirks[14]) { if (a_dst) { a_dst->meta.wi = ((size_t)(iop_a_dst - a_dst->data.ptr)); } if (a_src) { a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); } - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(16); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(17); status = wuffs_json__decoder__decode_inf_nan(self, a_dst, a_src); if (a_dst) { iop_a_dst = a_dst->data.ptr + a_dst->meta.wi; @@ -26420,10 +26473,10 @@ wuffs_json__decoder__decode_tokens( goto exit; } else { status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(17); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(18); while (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { status = wuffs_base__make_status(wuffs_base__suspension__short_write); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(18); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(19); } } } @@ -26539,7 +26592,7 @@ wuffs_json__decoder__decode_tokens( goto label__goto_parsed_a_leaf_value__break; } else if (v_match == 1) { status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(19); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(20); goto label__outer__continue; } } else if (v_class == 10) { @@ -26556,7 +26609,7 @@ wuffs_json__decoder__decode_tokens( goto label__goto_parsed_a_leaf_value__break; } else if (v_match == 1) { status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(20); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(21); goto label__outer__continue; } } else if (v_class == 11) { @@ -26573,17 +26626,17 @@ wuffs_json__decoder__decode_tokens( goto label__goto_parsed_a_leaf_value__break; } else if (v_match == 1) { status = wuffs_base__make_status(wuffs_base__suspension__short_read); - WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(21); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(22); goto label__outer__continue; } - if (self->private_impl.f_quirks[13]) { + if (self->private_impl.f_quirks[14]) { if (a_dst) { a_dst->meta.wi = ((size_t)(iop_a_dst - a_dst->data.ptr)); } if (a_src) { a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); } - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(22); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(23); status = wuffs_json__decoder__decode_inf_nan(self, a_dst, a_src); if (a_dst) { iop_a_dst = a_dst->data.ptr + a_dst->meta.wi; @@ -26597,14 +26650,14 @@ wuffs_json__decoder__decode_tokens( goto label__goto_parsed_a_leaf_value__break; } } else if (v_class == 12) { - if (self->private_impl.f_quirks[10] || self->private_impl.f_quirks[11]) { + if (self->private_impl.f_quirks[11] || self->private_impl.f_quirks[12]) { if (a_dst) { a_dst->meta.wi = ((size_t)(iop_a_dst - a_dst->data.ptr)); } if (a_src) { a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); } - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(23); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(24); status = wuffs_json__decoder__decode_comment(self, a_dst, a_src); if (a_dst) { iop_a_dst = a_dst->data.ptr + a_dst->meta.wi; @@ -26628,14 +26681,14 @@ wuffs_json__decoder__decode_tokens( v_expect = v_expect_after_value; } label__outer__break:; - if (self->private_impl.f_quirks[16]) { + if (self->private_impl.f_quirks[17]) { if (a_dst) { a_dst->meta.wi = ((size_t)(iop_a_dst - a_dst->data.ptr)); } if (a_src) { a_src->meta.ri = ((size_t)(iop_a_src - a_src->data.ptr)); } - WUFFS_BASE__COROUTINE_SUSPENSION_POINT(24); + WUFFS_BASE__COROUTINE_SUSPENSION_POINT(25); status = wuffs_json__decoder__decode_trailing_new_line(self, a_dst, a_src); if (a_dst) { iop_a_dst = a_dst->data.ptr + a_dst->meta.wi; @@ -26903,8 +26956,8 @@ wuffs_json__decoder__decode_leading( switch (coro_susp_point) { WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; - self->private_impl.f_allow_leading_ars = self->private_impl.f_quirks[14]; - self->private_impl.f_allow_leading_ubom = self->private_impl.f_quirks[15]; + self->private_impl.f_allow_leading_ars = self->private_impl.f_quirks[15]; + self->private_impl.f_allow_leading_ubom = self->private_impl.f_quirks[16]; label__0__continue:; while (self->private_impl.f_allow_leading_ars || self->private_impl.f_allow_leading_ubom) { if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { @@ -27029,7 +27082,7 @@ wuffs_json__decoder__decode_comment( WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(2); } v_c2 = wuffs_base__load_u16le__no_bounds_check(iop_a_src); - if ((v_c2 == 10799) && self->private_impl.f_quirks[10]) { + if ((v_c2 == 10799) && self->private_impl.f_quirks[11]) { (iop_a_src += 2, wuffs_base__make_empty_struct()); v_length = 2; label__comment_block__continue:; @@ -27078,7 +27131,7 @@ wuffs_json__decoder__decode_comment( v_length += 1; } } - } else if ((v_c2 == 12079) && self->private_impl.f_quirks[11]) { + } else if ((v_c2 == 12079) && self->private_impl.f_quirks[12]) { (iop_a_src += 2, wuffs_base__make_empty_struct()); v_length = 2; label__comment_line__continue:; diff --git a/script/print-json-token-debug-format.c b/script/print-json-token-debug-format.c index 03f07403..c9209b6b 100644 --- a/script/print-json-token-debug-format.c +++ b/script/print-json-token-debug-format.c @@ -290,7 +290,7 @@ main1(int argc, char** argv) { WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_QUESTION_MARK, WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_SINGLE_QUOTE, WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_V, - WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X, + WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X_AS_CODE_POINTS, WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_ZERO, WUFFS_JSON__QUIRK_ALLOW_COMMENT_BLOCK, WUFFS_JSON__QUIRK_ALLOW_COMMENT_LINE, diff --git a/std/json/common_consts.wuffs b/std/json/common_consts.wuffs index 7c5b445e..b0724d96 100644 --- a/std/json/common_consts.wuffs +++ b/std/json/common_consts.wuffs @@ -16,6 +16,7 @@ pub status "#bad C0 control code" pub status "#bad UTF-8" pub status "#bad backslash-escape" pub status "#bad input" +pub status "#bad quirk combination" pub status "#unsupported number length" pub status "#unsupported recursion depth" @@ -123,7 +124,7 @@ pri const LUT_BACKSLASHES : array[256] base.u8 = [ // LUT_QUIRKY_BACKSLASHES_QUIRKS is discussed in the LUT_BACKSLASHES comment. // The first element (index 0) is not used, but 8 is a round power of 2, so // enforcing index-in-bounds is a simple "&7" operation. -pri const LUT_QUIRKY_BACKSLASHES_QUIRKS : array[8] base.u8[..= 0x09] = [ +pri const LUT_QUIRKY_BACKSLASHES_QUIRKS : array[8] base.u8[..= 0x0A] = [ 0, // Uncommenting the following lines requires being able to define consts in // terms of other consts, which could be non-trivial to specify and @@ -134,7 +135,7 @@ pri const LUT_QUIRKY_BACKSLASHES_QUIRKS : array[8] base.u8[..= 0x09] = [ 0x05, // (QUIRK_ALLOW_BACKSLASH_QUESTION_MARK - QUIRKS_BASE) as base.u8, 0x06, // (QUIRK_ALLOW_BACKSLASH_SINGLE_QUOTE - QUIRKS_BASE) as base.u8, 0x07, // (QUIRK_ALLOW_BACKSLASH_V - QUIRKS_BASE) as base.u8, - 0x09, // (QUIRK_ALLOW_BACKSLASH_ZERO - QUIRKS_BASE) as base.u8, + 0x0A, // (QUIRK_ALLOW_BACKSLASH_ZERO - QUIRKS_BASE) as base.u8, ] // LUT_QUIRKY_BACKSLASHES_CHARS is discussed in the LUT_BACKSLASHES comment. diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs index 7407994b..2b770d13 100644 --- a/std/json/decode_json.wuffs +++ b/std/json/decode_json.wuffs @@ -20,6 +20,8 @@ pub struct decoder? implements base.token_decoder( end_of_data : base.bool, + string_value_minor : base.u32[..= 0x1FF_FFFF], + util : base.utility, )( // stack is conceptually an array of bits, implemented as an array of u32. @@ -92,6 +94,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor var backslash_x_length : base.u32[..= 0xFFFF] var backslash_x_ok : base.u8 + var backslash_x_value : base.u8 var backslash_x_string : base.u32 var uni4_ok : base.u8 @@ -121,6 +124,16 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor this.decode_leading?(dst: args.dst, src: args.src) } + if this.quirks[QUIRK_ALLOW_BACKSLASH_X_AS_BYTES - QUIRKS_BASE] { + if this.quirks[QUIRK_ALLOW_BACKSLASH_X_AS_CODE_POINTS - QUIRKS_BASE] { + return "#bad quirk combination" + } + this.string_value_minor = (base.TOKEN__VBC__STRING << 21) + } else { + this.string_value_minor = (base.TOKEN__VBC__STRING << 21) | + base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 + } + expect = EXPECT_VALUE while.outer true { @@ -204,9 +217,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor // Emit the leading '"'. args.dst.write_simple_token_fast!( value_major: 0, - value_minor: (base.TOKEN__VBC__STRING << 21) | + value_minor: this.string_value_minor | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | - base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__DEFINITELY_ASCII | base.TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP, continued: 1, @@ -227,9 +239,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor if string_length > 0 { args.dst.write_simple_token_fast!( value_major: 0, - value_minor: (base.TOKEN__VBC__STRING << 21) | + value_minor: this.string_value_minor | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | - base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: string_length) @@ -260,9 +271,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor if string_length > (0xFFFB - 4) { args.dst.write_simple_token_fast!( value_major: 0, - value_minor: (base.TOKEN__VBC__STRING << 21) | + value_minor: this.string_value_minor | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | - base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: string_length + 4) @@ -280,9 +290,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor if string_length >= 0xFFFB { args.dst.write_simple_token_fast!( value_major: 0, - value_minor: (base.TOKEN__VBC__STRING << 21) | + value_minor: this.string_value_minor | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | - base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: 0xFFFC) @@ -296,9 +305,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor if string_length <> 0 { args.dst.write_simple_token_fast!( value_major: 0, - value_minor: (base.TOKEN__VBC__STRING << 21) | + value_minor: this.string_value_minor | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | - base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: string_length) @@ -310,9 +318,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor if string_length > 0 { args.dst.write_simple_token_fast!( value_major: 0, - value_minor: (base.TOKEN__VBC__STRING << 21) | + value_minor: this.string_value_minor | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | - base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: string_length) @@ -559,53 +566,88 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor } // -------- END backslash-capital-u. - } else if (c == 'x') and - this.quirks[QUIRK_ALLOW_BACKSLASH_X - QUIRKS_BASE] { + } else if (c == 'x') { // -------- BEGIN backslash-x - if args.src.available() < 4 { - if args.src.is_closed() { - return "#bad backslash-escape" + if this.quirks[QUIRK_ALLOW_BACKSLASH_X_AS_CODE_POINTS - QUIRKS_BASE] { + if args.src.available() < 4 { + if args.src.is_closed() { + return "#bad backslash-escape" + } + yield? base."$short read" + string_length = 0 + char = 0 + continue.string_loop_outer } - yield? base."$short read" - string_length = 0 - char = 0 - continue.string_loop_outer - } - backslash_x_length = 0 - while (backslash_x_length <= 0xFFFB) and (args.src.available() >= 4), - inv args.dst.available() > 0, - { backslash_x_string = args.src.peek_u32le() backslash_x_ok = 0x80 c = LUT_HEXADECIMAL_DIGITS[0xFF & (backslash_x_string >> 16)] backslash_x_ok &= c + backslash_x_value = (c & 0x0F) << 4 c = LUT_HEXADECIMAL_DIGITS[0xFF & (backslash_x_string >> 24)] backslash_x_ok &= c + backslash_x_value |= (c & 0x0F) if (backslash_x_ok == 0) or ((backslash_x_string & 0xFFFF) <> 0x785C) { // It wasn't "\\x34", for some hexadecimal // digits "34". - break + return "#bad backslash-escape" } args.src.skip_u32_fast!(actual: 4, worst_case: 4) - backslash_x_length += 4 - } endwhile + args.dst.write_simple_token_fast!( + value_major: 0, + value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) | + (backslash_x_value as base.u32), + continued: 1, + length: 4) + continue.string_loop_outer + } else if this.quirks[QUIRK_ALLOW_BACKSLASH_X_AS_BYTES - QUIRKS_BASE] { + if args.src.available() < 4 { + if args.src.is_closed() { + return "#bad backslash-escape" + } + yield? base."$short read" + string_length = 0 + char = 0 + continue.string_loop_outer + } - if backslash_x_length == 0 { - return "#bad backslash-escape" + backslash_x_length = 0 + while (backslash_x_length <= 0xFFFB) and (args.src.available() >= 4), + inv args.dst.available() > 0, + { + backslash_x_string = args.src.peek_u32le() + backslash_x_ok = 0x80 + + c = LUT_HEXADECIMAL_DIGITS[0xFF & (backslash_x_string >> 16)] + backslash_x_ok &= c + c = LUT_HEXADECIMAL_DIGITS[0xFF & (backslash_x_string >> 24)] + backslash_x_ok &= c + + if (backslash_x_ok == 0) or + ((backslash_x_string & 0xFFFF) <> 0x785C) { + // It wasn't "\\x34", for some hexadecimal + // digits "34". + break + } + args.src.skip_u32_fast!(actual: 4, worst_case: 4) + backslash_x_length += 4 + } endwhile + + if backslash_x_length == 0 { + return "#bad backslash-escape" + } + args.dst.write_simple_token_fast!( + value_major: 0, + value_minor: this.string_value_minor | + base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | + base.TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X, + continued: 1, + length: backslash_x_length) + continue.string_loop_outer } - args.dst.write_simple_token_fast!( - value_major: 0, - value_minor: (base.TOKEN__VBC__STRING << 21) | - base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | - base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | - base.TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X, - continued: 1, - length: backslash_x_length) - continue.string_loop_outer // -------- END backslash-x } @@ -616,9 +658,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor if string_length > 0 { args.dst.write_simple_token_fast!( value_major: 0, - value_minor: (base.TOKEN__VBC__STRING << 21) | + value_minor: this.string_value_minor | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | - base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: string_length) @@ -653,9 +694,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor if string_length >= 0xFFF8 { args.dst.write_simple_token_fast!( value_major: 0, - value_minor: (base.TOKEN__VBC__STRING << 21) | + value_minor: this.string_value_minor | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | - base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: string_length + 2) @@ -671,9 +711,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor if string_length > 0 { args.dst.write_simple_token_fast!( value_major: 0, - value_minor: (base.TOKEN__VBC__STRING << 21) | + value_minor: this.string_value_minor | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | - base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: string_length) @@ -712,9 +751,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor if string_length >= 0xFFF8 { args.dst.write_simple_token_fast!( value_major: 0, - value_minor: (base.TOKEN__VBC__STRING << 21) | + value_minor: this.string_value_minor | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | - base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: string_length + 3) @@ -731,9 +769,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor if string_length > 0 { args.dst.write_simple_token_fast!( value_major: 0, - value_minor: (base.TOKEN__VBC__STRING << 21) | + value_minor: this.string_value_minor | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | - base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: string_length) @@ -771,9 +808,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor if string_length >= 0xFFF8 { args.dst.write_simple_token_fast!( value_major: 0, - value_minor: (base.TOKEN__VBC__STRING << 21) | + value_minor: this.string_value_minor | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | - base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: string_length + 4) @@ -789,9 +825,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor if string_length > 0 { args.dst.write_simple_token_fast!( value_major: 0, - value_minor: (base.TOKEN__VBC__STRING << 21) | + value_minor: this.string_value_minor | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | - base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY, continued: 1, length: string_length) @@ -843,9 +878,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.src.skip_u32_fast!(actual: 1, worst_case: 1) args.dst.write_simple_token_fast!( value_major: 0, - value_minor: (base.TOKEN__VBC__STRING << 21) | + value_minor: this.string_value_minor | base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 | - base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 | base.TOKEN__VBD__STRING__DEFINITELY_ASCII | base.TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP, continued: 0, diff --git a/std/json/decode_quirks.wuffs b/std/json/decode_quirks.wuffs index 946ab782..2d9a97c7 100644 --- a/std/json/decode_quirks.wuffs +++ b/std/json/decode_quirks.wuffs @@ -75,35 +75,35 @@ pub const QUIRK_ALLOW_BACKSLASH_SINGLE_QUOTE : base.u32 = 0x4909_9400 | 0x06 // character. pub const QUIRK_ALLOW_BACKSLASH_V : base.u32 = 0x4909_9400 | 0x07 -// When this quirk is enabled, e.g. "abc\xeFz" is accepted as a JSON string, -// There are exactly 2 encoded bytes after each "\x". +// When this quirk is enabled, e.g. "abc\xeFz" is accepted as a JSON string. It +// decodes to 5 bytes: 0x61, 0x62, 0x63, 0xEF and 0x7A. Decoded strings are +// byte strings, no longer guaranteed to be valid UTF-8 and even if the overall +// byte string is valid UTF-8, tokens are also no longer guaranteed to split on +// UTF-8 boundaries. // -// "\x", "\x9", "\x9$" and "\X99" are all still rejected. +// It is up to the caller whether and how to run UTF-8 validation. // -// It is up to the caller how to interpret the two hexadecimal digits. There -// are two sensible choices: as-a-byte or as-a-Unicode-code-point. +// There are exactly 2 encoded bytes after each "\x". "\x", "\x9", "\x9$" and +// "\X99" are all still rejected. // -// As-a-byte means decoding to 5 bytes: 0x61, 0x62, 0x63, 0xEF and 0x7A. -// Decoded strings are byte strings, no longer guaranteed to be valid UTF-8 and -// even if the overall byte string is valid UTF-8, tokens are also no longer -// guaranteed to split on UTF-8 boundaries. +// This quirk cannot be combined with QUIRK_ALLOW_BACKSLASH_X_AS_CODE_POINTS. +pub const QUIRK_ALLOW_BACKSLASH_X_AS_BYTES : base.u32 = 0x4909_9400 | 0x08 + +// When this quirk is enabled, e.g. "abc\xeFz" is accepted as a JSON string. It +// decodes to 6 bytes: 0x61, 0x62, 0x63, 0xC3, 0xAF and 0x7A. The UTF-8 +// encoding of U+00EF LATIN SMALL LETTER I WITH DIAERESIS is the two byte +// sequence (0xC3, 0xAF). Decoded strings are still valid UTF-8 and tokens +// still split on UTF-8 boundaries. // -// As-a-Unicode-code-point means decoding to 6 bytes: 0x61, 0x62, 0x63, 0xC3, -// 0xAF and 0x7A. The UTF-8 encoding of U+00EF LATIN SMALL LETTER I WITH -// DIAERESIS is the two byte sequence (0xC3, 0xAF). Decoded strings are still -// valid UTF-8 and tokens still split on UTF-8 boundaries. +// There are exactly 2 encoded bytes after each "\x". "\x", "\x9", "\x9$" and +// "\X99" are all still rejected. // -// The parser cannot know which interpretation the caller will choose, so it -// will still mark its string tokens with TOKEN__VBD__STRING__DEFINITELY_UTF_8 -// and TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8, like all other JSON string -// tokens, even though the as-a-byte interpretation can result in invalid -// UTF-8. If the caller chooses as-a-byte, it is up to the caller whether and -// how to run UTF-8 validation. -pub const QUIRK_ALLOW_BACKSLASH_X : base.u32 = 0x4909_9400 | 0x08 +// This quirk cannot be combined with QUIRK_ALLOW_BACKSLASH_X_AS_BYTES. +pub const QUIRK_ALLOW_BACKSLASH_X_AS_CODE_POINTS : base.u32 = 0x4909_9400 | 0x09 // When this quirk is enabled, e.g. "abc\0z" is accepted as a JSON string, // equivalent to "abc\u0000z", containing an ASCII NUL control character. -pub const QUIRK_ALLOW_BACKSLASH_ZERO : base.u32 = 0x4909_9400 | 0x09 +pub const QUIRK_ALLOW_BACKSLASH_ZERO : base.u32 = 0x4909_9400 | 0x0A // When this quirk is enabled, "/* C/C++ style block comments */" are accepted // anywhere whitespace would be, although see the QUIRK_ALLOW_TRAILING_NEW_LINE @@ -111,7 +111,7 @@ pub const QUIRK_ALLOW_BACKSLASH_ZERO : base.u32 = 0x4909_9400 | 0x09 // // They produce WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK tokens. The token // chain's source bytes includes the starting "/*" and the ending "*/". -pub const QUIRK_ALLOW_COMMENT_BLOCK : base.u32 = 0x4909_9400 | 0x0A +pub const QUIRK_ALLOW_COMMENT_BLOCK : base.u32 = 0x4909_9400 | 0x0B // When this quirk is enabled, "// C/C++ style line comments\n" are accepted // anywhere whitespace would be, although see the QUIRK_ALLOW_TRAILING_NEW_LINE @@ -126,7 +126,7 @@ pub const QUIRK_ALLOW_COMMENT_BLOCK : base.u32 = 0x4909_9400 | 0x0A // Even if the line comments are on consecutive lines, each line comment is a // separate token chain. There may be whitespace tokens between one line // comment's ending "\n" and the next one's starting "//". -pub const QUIRK_ALLOW_COMMENT_LINE : base.u32 = 0x4909_9400 | 0x0B +pub const QUIRK_ALLOW_COMMENT_LINE : base.u32 = 0x4909_9400 | 0x0C // When this quirk is enabled, there may be a comma after the final array // element or object key-value pair and before the closing "]" or "}". A comma @@ -135,12 +135,12 @@ pub const QUIRK_ALLOW_COMMENT_LINE : base.u32 = 0x4909_9400 | 0x0B // // For example, `[1,]`, `[1,2,3,]` and `{"k":"v",}` all become acceptable, but // `[,]`, `{,}` and `{"k",:"v"}` are still rejected. -pub const QUIRK_ALLOW_EXTRA_COMMA : base.u32 = 0x4909_9400 | 0x0C +pub const QUIRK_ALLOW_EXTRA_COMMA : base.u32 = 0x4909_9400 | 0x0D // When this quirk is enabled, "inf", "Infinity", "NAN" and their // case-insensitive variants, optionally preceded immediately by "-" or "+", // are accepted anywhere a JSON number would be. -pub const QUIRK_ALLOW_INF_NAN_NUMBERS : base.u32 = 0x4909_9400 | 0x0D +pub const QUIRK_ALLOW_INF_NAN_NUMBERS : base.u32 = 0x4909_9400 | 0x0E // When this quirk is enabled, the input byte stream may optionally start with // "\x1E" (the ASCII Record Separator control character). That byte is skipped @@ -151,7 +151,7 @@ pub const QUIRK_ALLOW_INF_NAN_NUMBERS : base.u32 = 0x4909_9400 | 0x0D // // When combined with QUIRK_ALLOW_TRAILING_NEW_LINE, this format is also known // as RFC 7464, Json Text Sequences and MIME type "application/json-seq". -pub const QUIRK_ALLOW_LEADING_ASCII_RECORD_SEPARATOR : base.u32 = 0x4909_9400 | 0x0E +pub const QUIRK_ALLOW_LEADING_ASCII_RECORD_SEPARATOR : base.u32 = 0x4909_9400 | 0x0F // When this quirk is enabled, the input byte stream may optionally start with // "\xEF\xBB\xBF", the UTF-8 encoding of the Unicode BOM (Byte Order Mark). @@ -159,7 +159,7 @@ pub const QUIRK_ALLOW_LEADING_ASCII_RECORD_SEPARATOR : base.u32 = 0x4909_9400 | // // When combined with QUIRK_ALLOW_LEADING_ASCII_RECORD_SEPARATOR, either mark // may come first in the byte stream. -pub const QUIRK_ALLOW_LEADING_UNICODE_BYTE_ORDER_MARK : base.u32 = 0x4909_9400 | 0x0F +pub const QUIRK_ALLOW_LEADING_UNICODE_BYTE_ORDER_MARK : base.u32 = 0x4909_9400 | 0x10 // When this quirk is enabled, following a successful decoding of a top-level // JSON value, any trailing whitespace (ASCII characters 0x09, 0x0A, 0x0D or @@ -195,7 +195,7 @@ pub const QUIRK_ALLOW_LEADING_UNICODE_BYTE_ORDER_MARK : base.u32 = 0x4909_9400 | // end-of-file or '\n' is encountered. Treating this as an error avoids any // ambiguity in accounting for new lines within a block comment or ending a // line comment. -pub const QUIRK_ALLOW_TRAILING_NEW_LINE : base.u32 = 0x4909_9400 | 0x10 +pub const QUIRK_ALLOW_TRAILING_NEW_LINE : base.u32 = 0x4909_9400 | 0x11 // When this quirk is enabled, invalid UTF-8 inside a JSON string is accepted. // Each byte of invalid UTF-8 is equivalent to "\uFFFD", the Unicode @@ -211,6 +211,6 @@ pub const QUIRK_ALLOW_TRAILING_NEW_LINE : base.u32 = 0x4909_9400 | 0x10 // When combined with QUIRK_ALLOW_BACKSLASH_CAPITAL_U, a "\U12345678" 10-byte // unit that is an invalid Unicode code point (i.e. in the range U+D800 ..= // U+DFFF or above U+10FFFF) is similarly replaced with U+FFFD. -pub const QUIRK_REPLACE_INVALID_UNICODE : base.u32 = 0x4909_9400 | 0x11 +pub const QUIRK_REPLACE_INVALID_UNICODE : base.u32 = 0x4909_9400 | 0x12 -pri const QUIRKS_COUNT : base.u32 = 0x12 +pri const QUIRKS_COUNT : base.u32 = 0x13 diff --git a/test/c/std/json.c b/test/c/std/json.c index df0d78be..e907f724 100644 --- a/test/c/std/json.c +++ b/test/c/std/json.c @@ -2371,7 +2371,7 @@ test_wuffs_json_decode_interface() { WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_QUESTION_MARK, WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_SINGLE_QUOTE, WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_V, - WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X, + WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X_AS_CODE_POINTS, WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_ZERO, WUFFS_JSON__QUIRK_ALLOW_COMMENT_BLOCK, WUFFS_JSON__QUIRK_ALLOW_COMMENT_LINE, @@ -2857,76 +2857,109 @@ test_wuffs_json_decode_quirk_allow_backslash_x() { struct { uint64_t want_bytes; + uint64_t want_code_points; const char* want_status_repr; const char* str; } test_cases[] = { - {.want_bytes = 0x12789A, + // U+009A in UTF-8 is C2 9A. + // U+00FF in UTF-8 is C3 BF. + // U+3456 in UTF-8 is E3 91 96. + {.want_bytes = 0x12E39196789A, + .want_code_points = 0x12E3919678C29A, .want_status_repr = NULL, .str = "\"\\x12\\u3456\\x78\\x9A\""}, + {.want_bytes = 0xFF, + .want_code_points = 0xC3BF, + .want_status_repr = NULL, + .str = "\"\\xFf\""}, {.want_bytes = 0x00, + .want_code_points = 0x00, .want_status_repr = wuffs_json__error__bad_backslash_escape, .str = "\"a\\X6A\""}, {.want_bytes = 0x6A6B, + .want_code_points = 0x6A6B, .want_status_repr = NULL, .str = "\"a\\x6A\\x6bz\""}, {.want_bytes = 0x6A, + .want_code_points = 0x6A, .want_status_repr = wuffs_json__error__bad_backslash_escape, .str = "\"a\\x6A\\x6yz\""}, {.want_bytes = 0x00, + .want_code_points = 0x00, .want_status_repr = wuffs_json__error__bad_backslash_escape, .str = "\"a\\x\""}, }; int tc; for (tc = 0; tc < WUFFS_TESTLIB_ARRAY_SIZE(test_cases); tc++) { - wuffs_json__decoder dec; - CHECK_STATUS("initialize", wuffs_json__decoder__initialize( - &dec, sizeof dec, WUFFS_VERSION, - WUFFS_INITIALIZE__DEFAULT_OPTIONS)); - wuffs_json__decoder__set_quirk_enabled( - &dec, WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X, true); + int q; + for (q = 0; q < 2; q++) { + wuffs_json__decoder dec; + CHECK_STATUS("initialize", wuffs_json__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, + WUFFS_INITIALIZE__DEFAULT_OPTIONS)); + wuffs_json__decoder__set_quirk_enabled( + &dec, + q ? WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X_AS_CODE_POINTS + : WUFFS_JSON__QUIRK_ALLOW_BACKSLASH_X_AS_BYTES, + true); - wuffs_base__token_buffer tok = - wuffs_base__slice_token__writer(g_have_slice_token); - wuffs_base__slice_u8 src_slice = wuffs_base__make_slice_u8( - (void*)(test_cases[tc].str), strlen(test_cases[tc].str)); - wuffs_base__io_buffer src = wuffs_base__slice_u8__reader(src_slice, true); - const char* have_status_repr = - wuffs_json__decoder__decode_tokens(&dec, &tok, &src, g_work_slice_u8) - .repr; - if (have_status_repr != test_cases[tc].want_status_repr) { - RETURN_FAIL("tc=%d: decode_tokens: have \"%s\", want \"%s\"", tc, - have_status_repr, test_cases[tc].want_status_repr); - } - - uint64_t src_index = 0; - uint64_t have_bytes = 0; - while (tok.meta.ri < tok.meta.wi) { - wuffs_base__token* t = &tok.data.ptr[tok.meta.ri++]; - int64_t vbc = wuffs_base__token__value_base_category(t); - uint64_t vbd = wuffs_base__token__value_base_detail(t); - uint64_t token_length = wuffs_base__token__length(t); - if ((vbc == WUFFS_BASE__TOKEN__VBC__STRING) && - (vbd & - WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X)) { - uint8_t b[8] = {0}; - const bool src_closed = true; - wuffs_base__transform__output o = wuffs_base__base_16__decode4( - wuffs_base__make_slice_u8(&b[0], 8), - wuffs_base__make_slice_u8(src_slice.ptr + src_index, token_length), - src_closed, 0); - size_t i = 0; - for (; i < o.num_dst; i++) { - have_bytes <<= 8; - have_bytes |= b[i]; - } + wuffs_base__token_buffer tok = + wuffs_base__slice_token__writer(g_have_slice_token); + wuffs_base__slice_u8 src_slice = wuffs_base__make_slice_u8( + (void*)(test_cases[tc].str), strlen(test_cases[tc].str)); + wuffs_base__io_buffer src = wuffs_base__slice_u8__reader(src_slice, true); + const char* have_status_repr = + wuffs_json__decoder__decode_tokens(&dec, &tok, &src, g_work_slice_u8) + .repr; + if (have_status_repr != test_cases[tc].want_status_repr) { + RETURN_FAIL("tc=%d, q=%d: decode_tokens: have \"%s\", want \"%s\"", tc, + q, have_status_repr, test_cases[tc].want_status_repr); } - src_index += token_length; - } - if (have_bytes != test_cases[tc].want_bytes) { - RETURN_FAIL("tc=%d: have U+%08" PRIX64 ", want U+%08" PRIX64, tc, - have_bytes, test_cases[tc].want_bytes); + uint64_t src_index = 0; + uint64_t have = 0; + while (tok.meta.ri < tok.meta.wi) { + wuffs_base__token* t = &tok.data.ptr[tok.meta.ri++]; + int64_t vbc = wuffs_base__token__value_base_category(t); + uint64_t vbd = wuffs_base__token__value_base_detail(t); + uint64_t token_length = wuffs_base__token__length(t); + if ((vbc == WUFFS_BASE__TOKEN__VBC__STRING) && + (vbd & + WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X)) { + uint8_t b[8] = {0}; + const bool src_closed = true; + wuffs_base__transform__output o = wuffs_base__base_16__decode4( + wuffs_base__make_slice_u8(&b[0], 8), + wuffs_base__make_slice_u8(src_slice.ptr + src_index, + token_length), + src_closed, 0); + size_t i = 0; + for (; i < o.num_dst; i++) { + have <<= 8; + have |= b[i]; + } + } else if (vbc == WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT) { + uint8_t b[WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL]; + size_t n = wuffs_base__utf_8__encode( + wuffs_base__make_slice_u8( + &b[0], WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL), + vbd); + size_t i = 0; + for (; i < n; i++) { + have <<= 8; + have |= b[i]; + } + } + + src_index += token_length; + } + uint64_t want = + q ? test_cases[tc].want_code_points : test_cases[tc].want_bytes; + if (have != want) { + RETURN_FAIL("tc=%d, q=%d: have U+%08" PRIX64 ", want U+%08" PRIX64, tc, + q, have, want); + } } } diff --git a/test/data/json-quirks.tokens b/test/data/json-quirks.tokens index 7569ad192c3cc139a74ea9328fa665c8f8177c86..3a36b054b5b3dd196dccc93d33108963143247dd 100644 GIT binary patch delta 39 mcmX@Wet~_1DYG;S10w?i0|PSyLj(f@qca17A3529xd8xjs0950 delta 23 ecmcb>et><0DKj?*10w?i0|OHS2lM0r<^}*c@B|J3