mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-21 08:47:28 +01:00
Add TOKEN__VBD__STRING__CHAIN_ETC
This commit is contained in:
1 parent
0eee231ce6
commit
b9afc26387
10 files changed
+117
-57
No files matched your search
@@ -82,15 +82,31 @@ wuffs_base__make_token(uint64_t repr) {
|
||||
|
||||
// --------
|
||||
|
||||
// "DEFINITELY_FOO" means that the destination bytes (and also the source
|
||||
// bytes, for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that
|
||||
// the lack of the bit is conservative: it is valid for all-ASCII strings to
|
||||
// have neither DEFINITELY_UTF_8 or DEFINITELY_ASCII bits set.
|
||||
// DEFINITELY_FOO means that the destination bytes (and also the source bytes,
|
||||
// for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that the lack
|
||||
// of the bit means "maybe FOO". It does not necessarily mean "not FOO".
|
||||
//
|
||||
// CHAIN_ETC means that decoding the entire token chain forms a UTF-8 or ASCII
|
||||
// string, not just this current token. CHAIN_ETC_UTF_8 therefore distinguishes
|
||||
// Unicode (UTF-8) strings from byte strings. MUST means that the the token
|
||||
// producer (e.g. parser) must verify this. SHOULD means that the token
|
||||
// consumer (e.g. renderer) should verify this.
|
||||
//
|
||||
// When a CHAIN_ETC_UTF_8 bit is set, the parser must ensure that non-ASCII
|
||||
// code points (with multi-byte UTF-8 encodings) do not straddle token
|
||||
// boundaries. Checking UTF-8 validity can inspect each token separately.
|
||||
//
|
||||
// The lack of any particular bit is conservative: it is valid for all-ASCII
|
||||
// strings, in a single- or multi-token chain, to have none of these bits set.
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00001
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00002
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 0x00002
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_SHOULD_BE_UTF_8 0x00004
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00010
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_ASCII 0x00020
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_SHOULD_BE_ASCII 0x00040
|
||||
|
||||
// "CONVERT_D_DST_S_SRC" means that multiples of S source bytes (possibly
|
||||
// padded) produces multiples of D destination bytes. For example,
|
||||
// CONVERT_D_DST_S_SRC means that multiples of S source bytes (possibly padded)
|
||||
// produces multiples of D destination bytes. For example,
|
||||
// CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like "\\x23\\x67\\xAB", where
|
||||
// 12 src bytes encode 3 dst bytes.
|
||||
//
|
||||
@@ -101,13 +117,13 @@ wuffs_base__make_token(uint64_t repr) {
|
||||
// When src is the empty string, multiple conversion algorithms are applicable
|
||||
// (so these bits are not necessarily mutually exclusive), all producing the
|
||||
// same empty dst string.
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00010
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00020
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00040
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00080
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x00100
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x00200
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x00400
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00100
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00200
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00400
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00800
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x01000
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x02000
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x04000
|
||||
|
||||
// --------
|
||||
|
||||
|
||||
@@ -490,8 +490,9 @@ const BaseTokenPublicH = "" +
|
||||
"" +
|
||||
"// --------\n\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000\n\n" +
|
||||
"" +
|
||||
"// --------\n\n// \"DEFINITELY_FOO\" means that the destination bytes (and also the source\n// bytes, for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that\n// the lack of the bit is conservative: it is valid for all-ASCII strings to\n// have neither DEFINITELY_UTF_8 or DEFINITELY_ASCII bits set.\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00002\n\n// \"CONVERT_D_DST_S_SRC\" means that multiples of S source bytes (possibly\n// padded) produces multiples of D destination bytes. For example,\n// CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like \"\\\\x23\\\\x67\\\\xAB\", where\n// 12 src bytes encode 3 dst bytes.\n//\n// Post-processing may further transform those D destination bytes (e.g. treat\n// \"\\\\xFF\" as the Unicode code point U+00FF instead of the byte 0xFF), but that\n// is out of scope of this VBD's semantics.\n//\n// When src is the empty string, multiple conversion algorithms are applicable\n// (so these bits are not necessarily mutual" +
|
||||
"ly exclusive), all producing the\n// same empty dst string.\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00080\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x00100\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x00200\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x00400\n\n" +
|
||||
"// --------\n\n// DEFINITELY_FOO means that the destination bytes (and also the source bytes,\n// for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that the lack\n// of the bit means \"maybe FOO\". It does not necessarily mean \"not FOO\".\n//\n// CHAIN_ETC means that decoding the entire token chain forms a UTF-8 or ASCII\n// string, not just this current token. CHAIN_ETC_UTF_8 therefore distinguishes\n// Unicode (UTF-8) strings from byte strings. MUST means that the the token\n// producer (e.g. parser) must verify this. SHOULD means that the token\n// consumer (e.g. renderer) should verify this.\n//\n// When a CHAIN_ETC_UTF_8 bit is set, the parser must ensure that non-ASCII\n// code points (with multi-byte UTF-8 encodings) do not straddle token\n// boundaries. Checking UTF-8 validity can inspect each token separately.\n//\n// The lack of any particular bit is conservative: it is valid for all-ASCII\n// strings, in a single- or multi-token chain, to have none of these bits set.\n#define WUFFS_BASE__TOKEN__VBD__STRING_" +
|
||||
"_DEFINITELY_UTF_8 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 0x00002\n#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_SHOULD_BE_UTF_8 0x00004\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_ASCII 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_SHOULD_BE_ASCII 0x00040\n\n// CONVERT_D_DST_S_SRC means that multiples of S source bytes (possibly padded)\n// produces multiples of D destination bytes. For example,\n// CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like \"\\\\x23\\\\x67\\\\xAB\", where\n// 12 src bytes encode 3 dst bytes.\n//\n// Post-processing may further transform those D destination bytes (e.g. treat\n// \"\\\\xFF\" as the Unicode code point U+00FF instead of the byte 0xFF), but that\n// is out of scope of this VBD's semantics.\n//\n// When src is the empty string, multiple conversion algorithms are applicable\n// (so these bits are not necessarily mutually exclusive), all producing the\n// same empty dst string.\n#define WU" +
|
||||
"FFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00100\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00200\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00400\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00800\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x01000\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x02000\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x04000\n\n" +
|
||||
"" +
|
||||
"// --------\n\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__UNDEFINED 0x00001\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__NULL 0x00002\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__FALSE 0x00004\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__TRUE 0x00008\n\n" +
|
||||
"" +
|
||||
|
||||
Reference in new issue
Block a user