mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-18 00:32:11 +01:00
Add TOKEN__VBD__STRING__CHAIN_ETC
This commit is contained in:
1 parent
0eee231ce6
commit
b9afc26387
10 files changed
+117
-57
No files matched your search
@@ -82,15 +82,31 @@ wuffs_base__make_token(uint64_t repr) {
|
||||
|
||||
// --------
|
||||
|
||||
// "DEFINITELY_FOO" means that the destination bytes (and also the source
|
||||
// bytes, for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that
|
||||
// the lack of the bit is conservative: it is valid for all-ASCII strings to
|
||||
// have neither DEFINITELY_UTF_8 or DEFINITELY_ASCII bits set.
|
||||
// DEFINITELY_FOO means that the destination bytes (and also the source bytes,
|
||||
// for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that the lack
|
||||
// of the bit means "maybe FOO". It does not necessarily mean "not FOO".
|
||||
//
|
||||
// CHAIN_ETC means that decoding the entire token chain forms a UTF-8 or ASCII
|
||||
// string, not just this current token. CHAIN_ETC_UTF_8 therefore distinguishes
|
||||
// Unicode (UTF-8) strings from byte strings. MUST means that the the token
|
||||
// producer (e.g. parser) must verify this. SHOULD means that the token
|
||||
// consumer (e.g. renderer) should verify this.
|
||||
//
|
||||
// When a CHAIN_ETC_UTF_8 bit is set, the parser must ensure that non-ASCII
|
||||
// code points (with multi-byte UTF-8 encodings) do not straddle token
|
||||
// boundaries. Checking UTF-8 validity can inspect each token separately.
|
||||
//
|
||||
// The lack of any particular bit is conservative: it is valid for all-ASCII
|
||||
// strings, in a single- or multi-token chain, to have none of these bits set.
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00001
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00002
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 0x00002
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_SHOULD_BE_UTF_8 0x00004
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00010
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_ASCII 0x00020
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_SHOULD_BE_ASCII 0x00040
|
||||
|
||||
// "CONVERT_D_DST_S_SRC" means that multiples of S source bytes (possibly
|
||||
// padded) produces multiples of D destination bytes. For example,
|
||||
// CONVERT_D_DST_S_SRC means that multiples of S source bytes (possibly padded)
|
||||
// produces multiples of D destination bytes. For example,
|
||||
// CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like "\\x23\\x67\\xAB", where
|
||||
// 12 src bytes encode 3 dst bytes.
|
||||
//
|
||||
@@ -101,13 +117,13 @@ wuffs_base__make_token(uint64_t repr) {
|
||||
// When src is the empty string, multiple conversion algorithms are applicable
|
||||
// (so these bits are not necessarily mutually exclusive), all producing the
|
||||
// same empty dst string.
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00010
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00020
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00040
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00080
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x00100
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x00200
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x00400
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00100
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00200
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00400
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00800
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x01000
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x02000
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x04000
|
||||
|
||||
// --------
|
||||
|
||||
|
||||
@@ -490,8 +490,9 @@ const BaseTokenPublicH = "" +
|
||||
"" +
|
||||
"// --------\n\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000\n\n" +
|
||||
"" +
|
||||
"// --------\n\n// \"DEFINITELY_FOO\" means that the destination bytes (and also the source\n// bytes, for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that\n// the lack of the bit is conservative: it is valid for all-ASCII strings to\n// have neither DEFINITELY_UTF_8 or DEFINITELY_ASCII bits set.\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00002\n\n// \"CONVERT_D_DST_S_SRC\" means that multiples of S source bytes (possibly\n// padded) produces multiples of D destination bytes. For example,\n// CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like \"\\\\x23\\\\x67\\\\xAB\", where\n// 12 src bytes encode 3 dst bytes.\n//\n// Post-processing may further transform those D destination bytes (e.g. treat\n// \"\\\\xFF\" as the Unicode code point U+00FF instead of the byte 0xFF), but that\n// is out of scope of this VBD's semantics.\n//\n// When src is the empty string, multiple conversion algorithms are applicable\n// (so these bits are not necessarily mutual" +
|
||||
"ly exclusive), all producing the\n// same empty dst string.\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00080\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x00100\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x00200\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x00400\n\n" +
|
||||
"// --------\n\n// DEFINITELY_FOO means that the destination bytes (and also the source bytes,\n// for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that the lack\n// of the bit means \"maybe FOO\". It does not necessarily mean \"not FOO\".\n//\n// CHAIN_ETC means that decoding the entire token chain forms a UTF-8 or ASCII\n// string, not just this current token. CHAIN_ETC_UTF_8 therefore distinguishes\n// Unicode (UTF-8) strings from byte strings. MUST means that the the token\n// producer (e.g. parser) must verify this. SHOULD means that the token\n// consumer (e.g. renderer) should verify this.\n//\n// When a CHAIN_ETC_UTF_8 bit is set, the parser must ensure that non-ASCII\n// code points (with multi-byte UTF-8 encodings) do not straddle token\n// boundaries. Checking UTF-8 validity can inspect each token separately.\n//\n// The lack of any particular bit is conservative: it is valid for all-ASCII\n// strings, in a single- or multi-token chain, to have none of these bits set.\n#define WUFFS_BASE__TOKEN__VBD__STRING_" +
|
||||
"_DEFINITELY_UTF_8 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 0x00002\n#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_SHOULD_BE_UTF_8 0x00004\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_ASCII 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_SHOULD_BE_ASCII 0x00040\n\n// CONVERT_D_DST_S_SRC means that multiples of S source bytes (possibly padded)\n// produces multiples of D destination bytes. For example,\n// CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like \"\\\\x23\\\\x67\\\\xAB\", where\n// 12 src bytes encode 3 dst bytes.\n//\n// Post-processing may further transform those D destination bytes (e.g. treat\n// \"\\\\xFF\" as the Unicode code point U+00FF instead of the byte 0xFF), but that\n// is out of scope of this VBD's semantics.\n//\n// When src is the empty string, multiple conversion algorithms are applicable\n// (so these bits are not necessarily mutually exclusive), all producing the\n// same empty dst string.\n#define WU" +
|
||||
"FFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00100\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00200\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00400\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00800\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x01000\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x02000\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x04000\n\n" +
|
||||
"" +
|
||||
"// --------\n\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__UNDEFINED 0x00001\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__NULL 0x00002\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__FALSE 0x00004\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__TRUE 0x00008\n\n" +
|
||||
"" +
|
||||
|
||||
+12
-8
@@ -101,15 +101,19 @@ var Consts = [...]struct {
|
||||
// ----
|
||||
|
||||
{t.IDU32, "0x00001", "TOKEN__VBD__STRING__DEFINITELY_UTF_8"},
|
||||
{t.IDU32, "0x00002", "TOKEN__VBD__STRING__DEFINITELY_ASCII"},
|
||||
{t.IDU32, "0x00002", "TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8"},
|
||||
{t.IDU32, "0x00004", "TOKEN__VBD__STRING__CHAIN_SHOULD_BE_UTF_8"},
|
||||
{t.IDU32, "0x00010", "TOKEN__VBD__STRING__DEFINITELY_ASCII"},
|
||||
{t.IDU32, "0x00020", "TOKEN__VBD__STRING__CHAIN_MUST_BE_ASCII"},
|
||||
{t.IDU32, "0x00040", "TOKEN__VBD__STRING__CHAIN_SHOULD_BE_ASCII"},
|
||||
|
||||
{t.IDU32, "0x00010", "TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP"},
|
||||
{t.IDU32, "0x00020", "TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY"},
|
||||
{t.IDU32, "0x00040", "TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL"},
|
||||
{t.IDU32, "0x00080", "TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X"},
|
||||
{t.IDU32, "0x00100", "TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD"},
|
||||
{t.IDU32, "0x00200", "TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL"},
|
||||
{t.IDU32, "0x00400", "TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85"},
|
||||
{t.IDU32, "0x00100", "TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP"},
|
||||
{t.IDU32, "0x00200", "TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY"},
|
||||
{t.IDU32, "0x00400", "TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL"},
|
||||
{t.IDU32, "0x00800", "TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X"},
|
||||
{t.IDU32, "0x01000", "TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD"},
|
||||
{t.IDU32, "0x02000", "TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL"},
|
||||
{t.IDU32, "0x04000", "TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85"},
|
||||
|
||||
// ----
|
||||
|
||||
|
||||
@@ -2141,15 +2141,31 @@ wuffs_base__make_token(uint64_t repr) {
|
||||
|
||||
// --------
|
||||
|
||||
// "DEFINITELY_FOO" means that the destination bytes (and also the source
|
||||
// bytes, for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that
|
||||
// the lack of the bit is conservative: it is valid for all-ASCII strings to
|
||||
// have neither DEFINITELY_UTF_8 or DEFINITELY_ASCII bits set.
|
||||
// DEFINITELY_FOO means that the destination bytes (and also the source bytes,
|
||||
// for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that the lack
|
||||
// of the bit means "maybe FOO". It does not necessarily mean "not FOO".
|
||||
//
|
||||
// CHAIN_ETC means that decoding the entire token chain forms a UTF-8 or ASCII
|
||||
// string, not just this current token. CHAIN_ETC_UTF_8 therefore distinguishes
|
||||
// Unicode (UTF-8) strings from byte strings. MUST means that the the token
|
||||
// producer (e.g. parser) must verify this. SHOULD means that the token
|
||||
// consumer (e.g. renderer) should verify this.
|
||||
//
|
||||
// When a CHAIN_ETC_UTF_8 bit is set, the parser must ensure that non-ASCII
|
||||
// code points (with multi-byte UTF-8 encodings) do not straddle token
|
||||
// boundaries. Checking UTF-8 validity can inspect each token separately.
|
||||
//
|
||||
// The lack of any particular bit is conservative: it is valid for all-ASCII
|
||||
// strings, in a single- or multi-token chain, to have none of these bits set.
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00001
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00002
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 0x00002
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_SHOULD_BE_UTF_8 0x00004
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00010
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_ASCII 0x00020
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_SHOULD_BE_ASCII 0x00040
|
||||
|
||||
// "CONVERT_D_DST_S_SRC" means that multiples of S source bytes (possibly
|
||||
// padded) produces multiples of D destination bytes. For example,
|
||||
// CONVERT_D_DST_S_SRC means that multiples of S source bytes (possibly padded)
|
||||
// produces multiples of D destination bytes. For example,
|
||||
// CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like "\\x23\\x67\\xAB", where
|
||||
// 12 src bytes encode 3 dst bytes.
|
||||
//
|
||||
@@ -2160,13 +2176,13 @@ wuffs_base__make_token(uint64_t repr) {
|
||||
// When src is the empty string, multiple conversion algorithms are applicable
|
||||
// (so these bits are not necessarily mutually exclusive), all producing the
|
||||
// same empty dst string.
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00010
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00020
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00040
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00080
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x00100
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x00200
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x00400
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00100
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00200
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00400
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00800
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x01000
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x02000
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x04000
|
||||
|
||||
// --------
|
||||
|
||||
@@ -23102,7 +23118,7 @@ wuffs_json__decoder__decode_tokens(
|
||||
}
|
||||
if (v_class == 1) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194323)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(4194579)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
(iop_a_src += 1, wuffs_base__make_empty_struct());
|
||||
@@ -23119,7 +23135,7 @@ wuffs_json__decoder__decode_tokens(
|
||||
if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) {
|
||||
if (v_string_length > 0) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -23144,7 +23160,7 @@ wuffs_json__decoder__decode_tokens(
|
||||
(iop_a_src += 4, wuffs_base__make_empty_struct());
|
||||
if (v_string_length > 65527) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)((v_string_length + 4))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -23159,7 +23175,7 @@ wuffs_json__decoder__decode_tokens(
|
||||
(iop_a_src += 1, wuffs_base__make_empty_struct());
|
||||
if (v_string_length >= 65531) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(65532)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -23170,7 +23186,7 @@ wuffs_json__decoder__decode_tokens(
|
||||
} else if (v_char == 1) {
|
||||
if (v_string_length != 0) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -23179,7 +23195,7 @@ wuffs_json__decoder__decode_tokens(
|
||||
} else if (v_char == 2) {
|
||||
if (v_string_length > 0) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -23405,7 +23421,7 @@ wuffs_json__decoder__decode_tokens(
|
||||
goto exit;
|
||||
}
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194432)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(4196355)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(v_backslash_x_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__string_loop_outer__continue;
|
||||
@@ -23416,7 +23432,7 @@ wuffs_json__decoder__decode_tokens(
|
||||
if (((uint64_t)(io2_a_src - iop_a_src)) < 2) {
|
||||
if (v_string_length > 0) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -23448,7 +23464,7 @@ wuffs_json__decoder__decode_tokens(
|
||||
(iop_a_src += 2, wuffs_base__make_empty_struct());
|
||||
if (v_string_length >= 65528) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)((v_string_length + 2))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -23461,7 +23477,7 @@ wuffs_json__decoder__decode_tokens(
|
||||
if (((uint64_t)(io2_a_src - iop_a_src)) < 3) {
|
||||
if (v_string_length > 0) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -23494,7 +23510,7 @@ wuffs_json__decoder__decode_tokens(
|
||||
(iop_a_src += 3, wuffs_base__make_empty_struct());
|
||||
if (v_string_length >= 65528) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)((v_string_length + 3))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -23508,7 +23524,7 @@ wuffs_json__decoder__decode_tokens(
|
||||
if (((uint64_t)(io2_a_src - iop_a_src)) < 4) {
|
||||
if (v_string_length > 0) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -23544,7 +23560,7 @@ wuffs_json__decoder__decode_tokens(
|
||||
(iop_a_src += 4, wuffs_base__make_empty_struct());
|
||||
if (v_string_length >= 65528) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)((v_string_length + 4))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -23557,7 +23573,7 @@ wuffs_json__decoder__decode_tokens(
|
||||
}
|
||||
if (v_string_length > 0) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(4194819)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -23608,7 +23624,7 @@ wuffs_json__decoder__decode_tokens(
|
||||
}
|
||||
(iop_a_src += 1, wuffs_base__make_empty_struct());
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194323)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(4194579)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__2__break;
|
||||
}
|
||||
|
||||
@@ -206,6 +206,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_ASCII |
|
||||
base.TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP,
|
||||
continued: 1,
|
||||
@@ -228,6 +229,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: string_length)
|
||||
@@ -260,6 +262,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: string_length + 4)
|
||||
@@ -279,6 +282,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: 0xFFFC)
|
||||
@@ -294,6 +298,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: string_length)
|
||||
@@ -307,6 +312,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: string_length)
|
||||
@@ -594,6 +600,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X,
|
||||
continued: 1,
|
||||
length: backslash_x_length)
|
||||
@@ -610,6 +618,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: string_length)
|
||||
@@ -646,6 +655,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: string_length + 2)
|
||||
@@ -663,6 +673,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: string_length)
|
||||
@@ -703,6 +714,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: string_length + 3)
|
||||
@@ -721,6 +733,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: string_length)
|
||||
@@ -760,6 +773,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: string_length + 4)
|
||||
@@ -777,6 +791,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: string_length)
|
||||
@@ -830,6 +845,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_ASCII |
|
||||
base.TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP,
|
||||
continued: 0,
|
||||
|
||||
@@ -81,17 +81,24 @@ pub const QUIRK_ALLOW_BACKSLASH_V : base.u32 = 0x4909_9400 | 0x07
|
||||
// "\x", "\x9", "\x9$" and "\X99" are all still rejected.
|
||||
//
|
||||
// It is up to the caller how to interpret the two hexadecimal digits. There
|
||||
// are two sensible choices: as a byte or as a Unicode code point.
|
||||
// are two sensible choices: as-a-byte or as-a-Unicode-code-point.
|
||||
//
|
||||
// As a byte means decoding to 5 bytes: 0x61, 0x62, 0x63, 0xEF and 0x7A.
|
||||
// As-a-byte means decoding to 5 bytes: 0x61, 0x62, 0x63, 0xEF and 0x7A.
|
||||
// Decoded strings are byte strings, no longer guaranteed to be valid UTF-8 and
|
||||
// even if the overall byte string is valid UTF-8, tokens are also no longer
|
||||
// guaranteed to split on UTF-8 boundaries.
|
||||
//
|
||||
// As a Unicode code point means decoding to 6 bytes: 0x61, 0x62, 0x63, 0xC3,
|
||||
// As-a-Unicode-code-point means decoding to 6 bytes: 0x61, 0x62, 0x63, 0xC3,
|
||||
// 0xAF and 0x7A. The UTF-8 encoding of U+00EF LATIN SMALL LETTER I WITH
|
||||
// DIAERESIS is the two byte sequence (0xC3, 0xAF). Decoded strings are still
|
||||
// valid UTF-8 and tokens still split on UTF-8 boundaries.
|
||||
//
|
||||
// The parser cannot know which interpretation the caller will choose, so it
|
||||
// will still mark its string tokens with TOKEN__VBD__STRING__DEFINITELY_UTF_8
|
||||
// and TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8, like all other JSON string
|
||||
// tokens, even though the as-a-byte interpretation can result in invalid
|
||||
// UTF-8. If the caller chooses as-a-byte, it is up to the caller whether and
|
||||
// how to run UTF-8 validation.
|
||||
pub const QUIRK_ALLOW_BACKSLASH_X : base.u32 = 0x4909_9400 | 0x08
|
||||
|
||||
// When this quirk is enabled, e.g. "abc\0z" is accepted as a JSON string,
|
||||
|
||||
+1
-1
@@ -2452,7 +2452,7 @@ test_wuffs_json_decode_quirk_allow_backslash_x() {
|
||||
uint64_t vbd = wuffs_base__token__value_base_detail(t);
|
||||
uint64_t token_length = wuffs_base__token__length(t);
|
||||
if ((vbc == WUFFS_BASE__TOKEN__VBC__STRING) &&
|
||||
(vbd ==
|
||||
(vbd &
|
||||
WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X)) {
|
||||
uint8_t b[8] = {0};
|
||||
size_t n = wuffs_base__hexadecimal__decode4(
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Reference in new issue
Block a user