Add TOKEN__VBD__STRING__CHAIN_ETC

This commit is contained in:
Nigel Tao committed 2020-07-03 13:36:13 +10:00
1 parent 0eee231ce6
commit b9afc26387
10 files changed
+117 -57

No files matched your search

+30 -14
View File
@@ -82,15 +82,31 @@ wuffs_base__make_token(uint64_t repr) {
// --------
// "DEFINITELY_FOO" means that the destination bytes (and also the source
// bytes, for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that
// the lack of the bit is conservative: it is valid for all-ASCII strings to
// have neither DEFINITELY_UTF_8 or DEFINITELY_ASCII bits set.
// DEFINITELY_FOO means that the destination bytes (and also the source bytes,
// for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that the lack
// of the bit means "maybe FOO". It does not necessarily mean "not FOO".
//
// CHAIN_ETC means that decoding the entire token chain forms a UTF-8 or ASCII
// string, not just this current token. CHAIN_ETC_UTF_8 therefore distinguishes
// Unicode (UTF-8) strings from byte strings. MUST means that the the token
// producer (e.g. parser) must verify this. SHOULD means that the token
// consumer (e.g. renderer) should verify this.
//
// When a CHAIN_ETC_UTF_8 bit is set, the parser must ensure that non-ASCII
// code points (with multi-byte UTF-8 encodings) do not straddle token
// boundaries. Checking UTF-8 validity can inspect each token separately.
//
// The lack of any particular bit is conservative: it is valid for all-ASCII
// strings, in a single- or multi-token chain, to have none of these bits set.
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00001
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00002
#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 0x00002
#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_SHOULD_BE_UTF_8 0x00004
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00010
#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_ASCII 0x00020
#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_SHOULD_BE_ASCII 0x00040
// "CONVERT_D_DST_S_SRC" means that multiples of S source bytes (possibly
// padded) produces multiples of D destination bytes. For example,
// CONVERT_D_DST_S_SRC means that multiples of S source bytes (possibly padded)
// produces multiples of D destination bytes. For example,
// CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like "\\x23\\x67\\xAB", where
// 12 src bytes encode 3 dst bytes.
//
@@ -101,13 +117,13 @@ wuffs_base__make_token(uint64_t repr) {
// When src is the empty string, multiple conversion algorithms are applicable
// (so these bits are not necessarily mutually exclusive), all producing the
// same empty dst string.
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00010
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00020
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00040
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00080
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x00100
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x00200
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x00400
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00100
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00200
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00400
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00800
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x01000
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x02000
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x04000
// --------
+3 -2
View File
@@ -490,8 +490,9 @@ const BaseTokenPublicH = "" +
"" +
"// --------\n\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000\n\n" +
"" +
"// --------\n\n// \"DEFINITELY_FOO\" means that the destination bytes (and also the source\n// bytes, for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that\n// the lack of the bit is conservative: it is valid for all-ASCII strings to\n// have neither DEFINITELY_UTF_8 or DEFINITELY_ASCII bits set.\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00002\n\n// \"CONVERT_D_DST_S_SRC\" means that multiples of S source bytes (possibly\n// padded) produces multiples of D destination bytes. For example,\n// CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like \"\\\\x23\\\\x67\\\\xAB\", where\n// 12 src bytes encode 3 dst bytes.\n//\n// Post-processing may further transform those D destination bytes (e.g. treat\n// \"\\\\xFF\" as the Unicode code point U+00FF instead of the byte 0xFF), but that\n// is out of scope of this VBD's semantics.\n//\n// When src is the empty string, multiple conversion algorithms are applicable\n// (so these bits are not necessarily mutual" +
"ly exclusive), all producing the\n// same empty dst string.\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00080\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x00100\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x00200\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x00400\n\n" +
"// --------\n\n// DEFINITELY_FOO means that the destination bytes (and also the source bytes,\n// for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that the lack\n// of the bit means \"maybe FOO\". It does not necessarily mean \"not FOO\".\n//\n// CHAIN_ETC means that decoding the entire token chain forms a UTF-8 or ASCII\n// string, not just this current token. CHAIN_ETC_UTF_8 therefore distinguishes\n// Unicode (UTF-8) strings from byte strings. MUST means that the the token\n// producer (e.g. parser) must verify this. SHOULD means that the token\n// consumer (e.g. renderer) should verify this.\n//\n// When a CHAIN_ETC_UTF_8 bit is set, the parser must ensure that non-ASCII\n// code points (with multi-byte UTF-8 encodings) do not straddle token\n// boundaries. Checking UTF-8 validity can inspect each token separately.\n//\n// The lack of any particular bit is conservative: it is valid for all-ASCII\n// strings, in a single- or multi-token chain, to have none of these bits set.\n#define WUFFS_BASE__TOKEN__VBD__STRING_" +
"_DEFINITELY_UTF_8 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 0x00002\n#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_SHOULD_BE_UTF_8 0x00004\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_ASCII 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_SHOULD_BE_ASCII 0x00040\n\n// CONVERT_D_DST_S_SRC means that multiples of S source bytes (possibly padded)\n// produces multiples of D destination bytes. For example,\n// CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like \"\\\\x23\\\\x67\\\\xAB\", where\n// 12 src bytes encode 3 dst bytes.\n//\n// Post-processing may further transform those D destination bytes (e.g. treat\n// \"\\\\xFF\" as the Unicode code point U+00FF instead of the byte 0xFF), but that\n// is out of scope of this VBD's semantics.\n//\n// When src is the empty string, multiple conversion algorithms are applicable\n// (so these bits are not necessarily mutually exclusive), all producing the\n// same empty dst string.\n#define WU" +
"FFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00100\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00200\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00400\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00800\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x01000\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x02000\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x04000\n\n" +
"" +
"// --------\n\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__UNDEFINED 0x00001\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__NULL 0x00002\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__FALSE 0x00004\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__TRUE 0x00008\n\n" +
"" +
+12 -8
View File
@@ -101,15 +101,19 @@ var Consts = [...]struct {
// ----
{t.IDU32, "0x00001", "TOKEN__VBD__STRING__DEFINITELY_UTF_8"},
{t.IDU32, "0x00002", "TOKEN__VBD__STRING__DEFINITELY_ASCII"},
{t.IDU32, "0x00002", "TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8"},
{t.IDU32, "0x00004", "TOKEN__VBD__STRING__CHAIN_SHOULD_BE_UTF_8"},
{t.IDU32, "0x00010", "TOKEN__VBD__STRING__DEFINITELY_ASCII"},
{t.IDU32, "0x00020", "TOKEN__VBD__STRING__CHAIN_MUST_BE_ASCII"},
{t.IDU32, "0x00040", "TOKEN__VBD__STRING__CHAIN_SHOULD_BE_ASCII"},
{t.IDU32, "0x00010", "TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP"},
{t.IDU32, "0x00020", "TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY"},
{t.IDU32, "0x00040", "TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL"},
{t.IDU32, "0x00080", "TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X"},
{t.IDU32, "0x00100", "TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD"},
{t.IDU32, "0x00200", "TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL"},
{t.IDU32, "0x00400", "TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85"},
{t.IDU32, "0x00100", "TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP"},
{t.IDU32, "0x00200", "TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY"},
{t.IDU32, "0x00400", "TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL"},
{t.IDU32, "0x00800", "TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X"},
{t.IDU32, "0x01000", "TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD"},
{t.IDU32, "0x02000", "TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL"},
{t.IDU32, "0x04000", "TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85"},
// ----
+45 -29
View File
@@ -2141,15 +2141,31 @@ wuffs_base__make_token(uint64_t repr) {
// --------
// "DEFINITELY_FOO" means that the destination bytes (and also the source
// bytes, for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that
// the lack of the bit is conservative: it is valid for all-ASCII strings to
// have neither DEFINITELY_UTF_8 or DEFINITELY_ASCII bits set.
// DEFINITELY_FOO means that the destination bytes (and also the source bytes,
// for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that the lack
// of the bit means "maybe FOO". It does not necessarily mean "not FOO".
//
// CHAIN_ETC means that decoding the entire token chain forms a UTF-8 or ASCII
// string, not just this current token. CHAIN_ETC_UTF_8 therefore distinguishes
// Unicode (UTF-8) strings from byte strings. MUST means that the the token
// producer (e.g. parser) must verify this. SHOULD means that the token
// consumer (e.g. renderer) should verify this.
//
// When a CHAIN_ETC_UTF_8 bit is set, the parser must ensure that non-ASCII
// code points (with multi-byte UTF-8 encodings) do not straddle token
// boundaries. Checking UTF-8 validity can inspect each token separately.
//
// The lack of any particular bit is conservative: it is valid for all-ASCII
// strings, in a single- or multi-token chain, to have none of these bits set.
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00001
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00002
#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 0x00002
#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_SHOULD_BE_UTF_8 0x00004
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00010
#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_ASCII 0x00020
#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_SHOULD_BE_ASCII 0x00040
// "CONVERT_D_DST_S_SRC" means that multiples of S source bytes (possibly
// padded) produces multiples of D destination bytes. For example,
// CONVERT_D_DST_S_SRC means that multiples of S source bytes (possibly padded)
// produces multiples of D destination bytes. For example,
// CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like "\\x23\\x67\\xAB", where
// 12 src bytes encode 3 dst bytes.
//
@@ -2160,13 +2176,13 @@ wuffs_base__make_token(uint64_t repr) {
// When src is the empty string, multiple conversion algorithms are applicable
// (so these bits are not necessarily mutually exclusive), all producing the
// same empty dst string.
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00010
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00020
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00040
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00080
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x00100
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x00200
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x00400
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00100
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00200
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00400
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00800
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x01000
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x02000
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x04000
// --------
@@ -23102,7 +23118,7 @@ wuffs_json__decoder__decode_tokens(
}
if (v_class == 1) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194323)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(4194579)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
(iop_a_src += 1, wuffs_base__make_empty_struct());
@@ -23119,7 +23135,7 @@ wuffs_json__decoder__decode_tokens(
if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) {
if (v_string_length > 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -23144,7 +23160,7 @@ wuffs_json__decoder__decode_tokens(
(iop_a_src += 4, wuffs_base__make_empty_struct());
if (v_string_length > 65527) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)((v_string_length + 4))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -23159,7 +23175,7 @@ wuffs_json__decoder__decode_tokens(
(iop_a_src += 1, wuffs_base__make_empty_struct());
if (v_string_length >= 65531) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(65532)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -23170,7 +23186,7 @@ wuffs_json__decoder__decode_tokens(
} else if (v_char == 1) {
if (v_string_length != 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -23179,7 +23195,7 @@ wuffs_json__decoder__decode_tokens(
} else if (v_char == 2) {
if (v_string_length > 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -23405,7 +23421,7 @@ wuffs_json__decoder__decode_tokens(
goto exit;
}
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194432)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(4196355)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(v_backslash_x_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__string_loop_outer__continue;
@@ -23416,7 +23432,7 @@ wuffs_json__decoder__decode_tokens(
if (((uint64_t)(io2_a_src - iop_a_src)) < 2) {
if (v_string_length > 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -23448,7 +23464,7 @@ wuffs_json__decoder__decode_tokens(
(iop_a_src += 2, wuffs_base__make_empty_struct());
if (v_string_length >= 65528) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)((v_string_length + 2))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -23461,7 +23477,7 @@ wuffs_json__decoder__decode_tokens(
if (((uint64_t)(io2_a_src - iop_a_src)) < 3) {
if (v_string_length > 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -23494,7 +23510,7 @@ wuffs_json__decoder__decode_tokens(
(iop_a_src += 3, wuffs_base__make_empty_struct());
if (v_string_length >= 65528) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)((v_string_length + 3))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -23508,7 +23524,7 @@ wuffs_json__decoder__decode_tokens(
if (((uint64_t)(io2_a_src - iop_a_src)) < 4) {
if (v_string_length > 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -23544,7 +23560,7 @@ wuffs_json__decoder__decode_tokens(
(iop_a_src += 4, wuffs_base__make_empty_struct());
if (v_string_length >= 65528) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)((v_string_length + 4))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -23557,7 +23573,7 @@ wuffs_json__decoder__decode_tokens(
}
if (v_string_length > 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
(((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
@@ -23608,7 +23624,7 @@ wuffs_json__decoder__decode_tokens(
}
(iop_a_src += 1, wuffs_base__make_empty_struct());
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194323)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(4194579)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__2__break;
}
+16
View File
@@ -206,6 +206,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
value_major: 0,
value_minor: (base.TOKEN__VBC__STRING << 21) |
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
base.TOKEN__VBD__STRING__DEFINITELY_ASCII |
base.TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP,
continued: 1,
@@ -228,6 +229,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
value_major: 0,
value_minor: (base.TOKEN__VBC__STRING << 21) |
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
continued: 1,
length: string_length)
@@ -260,6 +262,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
value_major: 0,
value_minor: (base.TOKEN__VBC__STRING << 21) |
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
continued: 1,
length: string_length + 4)
@@ -279,6 +282,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
value_major: 0,
value_minor: (base.TOKEN__VBC__STRING << 21) |
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
continued: 1,
length: 0xFFFC)
@@ -294,6 +298,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
value_major: 0,
value_minor: (base.TOKEN__VBC__STRING << 21) |
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
continued: 1,
length: string_length)
@@ -307,6 +312,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
value_major: 0,
value_minor: (base.TOKEN__VBC__STRING << 21) |
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
continued: 1,
length: string_length)
@@ -594,6 +600,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
args.dst.write_simple_token_fast!(
value_major: 0,
value_minor: (base.TOKEN__VBC__STRING << 21) |
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
base.TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X,
continued: 1,
length: backslash_x_length)
@@ -610,6 +618,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
value_major: 0,
value_minor: (base.TOKEN__VBC__STRING << 21) |
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
continued: 1,
length: string_length)
@@ -646,6 +655,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
value_major: 0,
value_minor: (base.TOKEN__VBC__STRING << 21) |
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
continued: 1,
length: string_length + 2)
@@ -663,6 +673,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
value_major: 0,
value_minor: (base.TOKEN__VBC__STRING << 21) |
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
continued: 1,
length: string_length)
@@ -703,6 +714,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
value_major: 0,
value_minor: (base.TOKEN__VBC__STRING << 21) |
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
continued: 1,
length: string_length + 3)
@@ -721,6 +733,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
value_major: 0,
value_minor: (base.TOKEN__VBC__STRING << 21) |
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
continued: 1,
length: string_length)
@@ -760,6 +773,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
value_major: 0,
value_minor: (base.TOKEN__VBC__STRING << 21) |
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
continued: 1,
length: string_length + 4)
@@ -777,6 +791,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
value_major: 0,
value_minor: (base.TOKEN__VBC__STRING << 21) |
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
continued: 1,
length: string_length)
@@ -830,6 +845,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
value_major: 0,
value_minor: (base.TOKEN__VBC__STRING << 21) |
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
base.TOKEN__VBD__STRING__DEFINITELY_ASCII |
base.TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP,
continued: 0,
+10 -3
View File
@@ -81,17 +81,24 @@ pub const QUIRK_ALLOW_BACKSLASH_V : base.u32 = 0x4909_9400 | 0x07
// "\x", "\x9", "\x9$" and "\X99" are all still rejected.
//
// It is up to the caller how to interpret the two hexadecimal digits. There
// are two sensible choices: as a byte or as a Unicode code point.
// are two sensible choices: as-a-byte or as-a-Unicode-code-point.
//
// As a byte means decoding to 5 bytes: 0x61, 0x62, 0x63, 0xEF and 0x7A.
// As-a-byte means decoding to 5 bytes: 0x61, 0x62, 0x63, 0xEF and 0x7A.
// Decoded strings are byte strings, no longer guaranteed to be valid UTF-8 and
// even if the overall byte string is valid UTF-8, tokens are also no longer
// guaranteed to split on UTF-8 boundaries.
//
// As a Unicode code point means decoding to 6 bytes: 0x61, 0x62, 0x63, 0xC3,
// As-a-Unicode-code-point means decoding to 6 bytes: 0x61, 0x62, 0x63, 0xC3,
// 0xAF and 0x7A. The UTF-8 encoding of U+00EF LATIN SMALL LETTER I WITH
// DIAERESIS is the two byte sequence (0xC3, 0xAF). Decoded strings are still
// valid UTF-8 and tokens still split on UTF-8 boundaries.
//
// The parser cannot know which interpretation the caller will choose, so it
// will still mark its string tokens with TOKEN__VBD__STRING__DEFINITELY_UTF_8
// and TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8, like all other JSON string
// tokens, even though the as-a-byte interpretation can result in invalid
// UTF-8. If the caller chooses as-a-byte, it is up to the caller whether and
// how to run UTF-8 validation.
pub const QUIRK_ALLOW_BACKSLASH_X : base.u32 = 0x4909_9400 | 0x08
// When this quirk is enabled, e.g. "abc\0z" is accepted as a JSON string,
+1 -1
View File
@@ -2452,7 +2452,7 @@ test_wuffs_json_decode_quirk_allow_backslash_x() {
uint64_t vbd = wuffs_base__token__value_base_detail(t);
uint64_t token_length = wuffs_base__token__length(t);
if ((vbc == WUFFS_BASE__TOKEN__VBC__STRING) &&
(vbd ==
(vbd &
WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X)) {
uint8_t b[8] = {0};
size_t n = wuffs_base__hexadecimal__decode4(
Binary file not shown.
Binary file not shown.
Binary file not shown.