Tweak wuffs_base__token bitmask assignments

This commit is contained in:
Nigel Tao committed 2020-02-25 12:12:31 +11:00
1 parent 16b0c4652a
commit a9d1488674
4 files changed
+247 -147

No files matched your search

+93 -44
View File
@@ -16,51 +16,23 @@
// ---------------- Tokens
#define WUFFS_BASE__TOKEN__VALUE__MASK 0xFFFFFFFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0xFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x7FFFFFF
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF
#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF
#define WUFFS_BASE__TOKEN__VALUE__SHIFT 16
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 40
#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 16
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 37
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16
#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0
#define WUFFS_BASE__TOKEN__VBC__FILLER 0
#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1
#define WUFFS_BASE__TOKEN__VBC__NUMBER 2
#define WUFFS_BASE__TOKEN__VBC__STRING 3
#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 4
#define WUFFS_BASE__TOKEN__VBC__BYTES 5
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL 0x00001
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__UNDEFINED 0x00011
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__NULL 0x00021
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__FALSE 0x00041
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__TRUE 0x00081
#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER 0x00002
#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00004
#define WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE 0x00001
#define WUFFS_BASE__TOKEN__VBD__STRING__ALL_ASCII 0x00002
typedef struct {
// Bits 63 .. 40 (24 bits) is the major value.
// Bits 39 .. 16 (24 bits) is the minor value.
// Bits 15 .. 0 (16 bits) is the length.
// The repr is divided as:
// - Bits 63 .. 40 (24 bits) is the major value.
// - Bits 39 .. 16 (24 bits) is the minor value.
// - Bits 15 .. 0 (16 bits) is the length.
//
// The major value is a [Base38](doc/note/base38-and-fourcc.md) value. If
// zero (special cased for Wuffs' built-in "base" package) then the minor
// value is further sub-divided:
// - Bits 39 .. 37 ( 3 bits) is the value_base_category.
// - Bits 36 .. 16 (21 bits) is the value_base_detail.
//
// In particular, at 21 bits, the value_base_detail can hold every valid
// Unicode code point.
//
// If the major value is non-zero then the minor value has whatever arbitrary
// meaning the tokenizer's package assigns to it.
uint64_t repr;
#ifdef __cplusplus
@@ -81,6 +53,83 @@ wuffs_base__make_token(uint64_t repr) {
return ret;
}
#define WUFFS_BASE__TOKEN__VALUE__MASK 0xFFFFFFFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0xFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x7FFFFFF
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF
#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF
#define WUFFS_BASE__TOKEN__VALUE__SHIFT 16
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 40
#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 16
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 37
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16
#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0
#define WUFFS_BASE__TOKEN__VBC__FILLER 0
#define WUFFS_BASE__TOKEN__VBC__STRING 1
#define WUFFS_BASE__TOKEN__VBC__BYTES 2
#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 3
#define WUFFS_BASE__TOKEN__VBC__NUMBER 4
#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 5
// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between
// VBD_FILLER, VBD_STRING and VBD_BYTES.
#define WUFFS_BASE__TOKEN__VBD__FILLER__INCOMPLETE 0x00001
#define WUFFS_BASE__TOKEN__VBD__FILLER__END_OF_CONSECUTIVE_COMMENTS 0x00010
#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00020
#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00040
// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between
// VBD_FILLER, VBD_STRING and VBD_BYTES.
#define WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE 0x00001
#define WUFFS_BASE__TOKEN__VBD__STRING__ALL_ASCII 0x00010
// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between
// VBD_FILLER, VBD_STRING and VBD_BYTES.
#define WUFFS_BASE__TOKEN__VBD__BYTES__INCOMPLETE 0x00001
// "D_DST_S_SRC" means that it takes S source bytes (possibly padded) to
// produce D destination bytes. For example,
// WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_4_SRC_BACKSLASH_X means that the source
// looks like "\\x23\\x67\\xAB", where 12 src bytes encode 3 dst bytes.
#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_1_SRC_RAW 0x00010
#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_2_SRC_HEX 0x00020
#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_4_SRC_BACKSLASH_X 0x00040
#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_6_SRC_BACKSLASH_U 0x00080
#define WUFFS_BASE__TOKEN__VBD__BYTES__3_DST_4_SRC_BASE_64_STD 0x00100
#define WUFFS_BASE__TOKEN__VBD__BYTES__3_DST_4_SRC_BASE_64_URL 0x00200
#define WUFFS_BASE__TOKEN__VBD__BYTES__4_DST_5_SRC_ASCII_85 0x00400
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL 0x00001
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__UNDEFINED 0x00101
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__NULL 0x00201
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__FALSE 0x00401
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__TRUE 0x00801
// For a source string of "123" or "0x9A", it is valid for a tokenizer to
// return any one of:
// - WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT.
// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED.
// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED.
//
// For a source string of "+123" or "-0x9A", only the first two are valid.
//
// For a source string of "123.", only the first one is valid.
#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00002
#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED 0x00004
#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED 0x00008
#define WUFFS_BASE__TOKEN__VBD__UNICODE_CODE_POINT__MAX_INCL 0x10FFFF
static inline uint64_t //
wuffs_base__token__value(const wuffs_base__token* t) {
return (t->repr >> WUFFS_BASE__TOKEN__VALUE__SHIFT) &
+7 -5
View File
@@ -284,11 +284,13 @@ const baseTokenPrivateH = "" +
""
const baseTokenPublicH = "" +
"// ---------------- Tokens\n\n#define WUFFS_BASE__TOKEN__VALUE__MASK 0xFFFFFFFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x7FFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF\n#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 16\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 40\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 16\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 37\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1\n#define WUFFS_BASE__TOKEN__VBC__NUMBER 2\n#define WUFFS_BASE__TOKEN__VBC__STRING 3\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 4\n#define WUFFS_BASE__TOKEN__VBC__BYTES 5\n\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x" +
"00002\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000\n\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL 0x00001\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__UNDEFINED 0x00011\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__NULL 0x00021\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__FALSE 0x00041\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__TRUE 0x00081\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER 0x00002\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00004\n\n#define WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRING__ALL_ASCII 0x00002\n\ntypedef struct {\n // Bits 63 .. 40 (24 bits) is the major value.\n // Bits 39 .. 16 (24 bits) is the minor v" +
"alue.\n // Bits 15 .. 0 (16 bits) is the length.\n uint64_t repr;\n\n#ifdef __cplusplus\n inline uint64_t value() const;\n inline uint64_t value_major() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_category() const;\n inline uint64_t value_base_detail() const;\n inline uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_major(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_minor(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_M" +
"INOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MINOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_category(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_detail(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__length(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) &\n WUFFS_BASE__TOKEN__LENGTH__MASK;\n}\n\n#ifdef __cplusplus\n\ninline uint64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_major() const {\n return wuffs_base__token__value_major(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(t" +
"his);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__token__value_base_detail(this);\n}\n\ninline uint64_t //\nwuffs_base__token::length() const {\n return wuffs_base__token__length(this);\n}\n\n#endif // __cplusplus\n\n" +
"// ---------------- Tokens\n\ntypedef struct {\n // The repr is divided as:\n // - Bits 63 .. 40 (24 bits) is the major value.\n // - Bits 39 .. 16 (24 bits) is the minor value.\n // - Bits 15 .. 0 (16 bits) is the length.\n //\n // The major value is a [Base38](doc/note/base38-and-fourcc.md) value. If\n // zero (special cased for Wuffs' built-in \"base\" package) then the minor\n // value is further sub-divided:\n // - Bits 39 .. 37 ( 3 bits) is the value_base_category.\n // - Bits 36 .. 16 (21 bits) is the value_base_detail.\n //\n // In particular, at 21 bits, the value_base_detail can hold every valid\n // Unicode code point.\n //\n // If the major value is non-zero then the minor value has whatever arbitrary\n // meaning the tokenizer's package assigns to it.\n uint64_t repr;\n\n#ifdef __cplusplus\n inline uint64_t value() const;\n inline uint64_t value_major() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_category() const;\n inline uint64_t value_base_detail() const;\n inlin" +
"e uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\n#define WUFFS_BASE__TOKEN__VALUE__MASK 0xFFFFFFFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x7FFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF\n#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 16\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 40\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 16\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 37\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRING 1\n#define WUFFS_BASE__TOKEN__VBC__BYTES 2\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 3\n#define WUFFS_BASE" +
"__TOKEN__VBC__NUMBER 4\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 5\n\n// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between\n// VBD_FILLER, VBD_STRING and VBD_BYTES.\n#define WUFFS_BASE__TOKEN__VBD__FILLER__INCOMPLETE 0x00001\n#define WUFFS_BASE__TOKEN__VBD__FILLER__END_OF_CONSECUTIVE_COMMENTS 0x00010\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00020\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00040\n\n// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between\n// VBD_FILLER, VBD_STRING and VBD_BYTES.\n#define WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRING__ALL_ASCII 0x00010\n\n// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between\n// VBD_FILLER, VBD_STRING and VBD_BYTES.\n#define WUFFS_BASE__TOKEN__VBD__BYTES__INCOMPLETE 0x00001\n// \"D_DST_S_SRC\" means that it takes S source bytes (possibly padded) to\n// produce D destination bytes. For example,\n// WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_4_SRC_BACKSLASH" +
"_X means that the source\n// looks like \"\\\\x23\\\\x67\\\\xAB\", where 12 src bytes encode 3 dst bytes.\n#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_1_SRC_RAW 0x00010\n#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_2_SRC_HEX 0x00020\n#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_4_SRC_BACKSLASH_X 0x00040\n#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_6_SRC_BACKSLASH_U 0x00080\n#define WUFFS_BASE__TOKEN__VBD__BYTES__3_DST_4_SRC_BASE_64_STD 0x00100\n#define WUFFS_BASE__TOKEN__VBD__BYTES__3_DST_4_SRC_BASE_64_URL 0x00200\n#define WUFFS_BASE__TOKEN__VBD__BYTES__4_DST_5_SRC_ASCII_85 0x00400\n\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000\n#define WUFFS_BASE__TOKEN__VBD__STRU" +
"CTURE__TO_DICT 0x04000\n\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL 0x00001\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__UNDEFINED 0x00101\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__NULL 0x00201\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__FALSE 0x00401\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__TRUE 0x00801\n// For a source string of \"123\" or \"0x9A\", it is valid for a tokenizer to\n// return any one of:\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT.\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED.\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED.\n//\n// For a source string of \"+123\" or \"-0x9A\", only the first two are valid.\n//\n// For a source string of \"123.\", only the first one is valid.\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00002\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED 0x00004\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED 0x00008\n\n#define WUFFS_BASE__TOKEN__VBD__UNICODE_CODE_POINT__MAX_INCL 0x10FFFF\n\nstatic inline uint64_t " +
" //\nwuffs_base__token__value(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_major(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_minor(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MINOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_category(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_detail(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK;\n}\n\nstatic inline uint64_t " +
" //\nwuffs_base__token__length(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) &\n WUFFS_BASE__TOKEN__LENGTH__MASK;\n}\n\n#ifdef __cplusplus\n\ninline uint64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_major() const {\n return wuffs_base__token__value_major(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__token__value_base_detail(this);\n}\n\ninline uint64_t //\nwuffs_base__token::length() const {\n return wuffs_base__token__length(this);\n}\n\n#endif // __cplusplus\n\n" +
"" +
"// --------\n\ntypedef WUFFS_BASE__SLICE(wuffs_base__token) wuffs_base__slice_token;\n\nstatic inline wuffs_base__slice_token //\nwuffs_base__make_slice_token(wuffs_base__token* ptr, size_t len) {\n wuffs_base__slice_token ret;\n ret.ptr = ptr;\n ret.len = len;\n return ret;\n}\n\n" +
"" +
+121 -72
View File
@@ -1476,53 +1476,25 @@ wuffs_base__io_buffer::writer_io_position() const {
#endif // __cplusplus
// ---------------- Tokens
#define WUFFS_BASE__TOKEN__VALUE__MASK 0xFFFFFFFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0xFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x7FFFFFF
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF
#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF
#define WUFFS_BASE__TOKEN__VALUE__SHIFT 16
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 40
#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 16
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 37
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16
#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0
#define WUFFS_BASE__TOKEN__VBC__FILLER 0
#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1
#define WUFFS_BASE__TOKEN__VBC__NUMBER 2
#define WUFFS_BASE__TOKEN__VBC__STRING 3
#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 4
#define WUFFS_BASE__TOKEN__VBC__BYTES 5
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL 0x00001
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__UNDEFINED 0x00011
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__NULL 0x00021
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__FALSE 0x00041
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__TRUE 0x00081
#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER 0x00002
#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00004
#define WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE 0x00001
#define WUFFS_BASE__TOKEN__VBD__STRING__ALL_ASCII 0x00002
// ---------------- Tokens
typedef struct {
// Bits 63 .. 40 (24 bits) is the major value.
// Bits 39 .. 16 (24 bits) is the minor value.
// Bits 15 .. 0 (16 bits) is the length.
// The repr is divided as:
// - Bits 63 .. 40 (24 bits) is the major value.
// - Bits 39 .. 16 (24 bits) is the minor value.
// - Bits 15 .. 0 (16 bits) is the length.
//
// The major value is a [Base38](doc/note/base38-and-fourcc.md) value. If
// zero (special cased for Wuffs' built-in "base" package) then the minor
// value is further sub-divided:
// - Bits 39 .. 37 ( 3 bits) is the value_base_category.
// - Bits 36 .. 16 (21 bits) is the value_base_detail.
//
// In particular, at 21 bits, the value_base_detail can hold every valid
// Unicode code point.
//
// If the major value is non-zero then the minor value has whatever arbitrary
// meaning the tokenizer's package assigns to it.
uint64_t repr;
#ifdef __cplusplus
@@ -1543,6 +1515,83 @@ wuffs_base__make_token(uint64_t repr) {
return ret;
}
#define WUFFS_BASE__TOKEN__VALUE__MASK 0xFFFFFFFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0xFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x7FFFFFF
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF
#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF
#define WUFFS_BASE__TOKEN__VALUE__SHIFT 16
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 40
#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 16
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 37
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16
#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0
#define WUFFS_BASE__TOKEN__VBC__FILLER 0
#define WUFFS_BASE__TOKEN__VBC__STRING 1
#define WUFFS_BASE__TOKEN__VBC__BYTES 2
#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 3
#define WUFFS_BASE__TOKEN__VBC__NUMBER 4
#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 5
// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between
// VBD_FILLER, VBD_STRING and VBD_BYTES.
#define WUFFS_BASE__TOKEN__VBD__FILLER__INCOMPLETE 0x00001
#define WUFFS_BASE__TOKEN__VBD__FILLER__END_OF_CONSECUTIVE_COMMENTS 0x00010
#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00020
#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00040
// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between
// VBD_FILLER, VBD_STRING and VBD_BYTES.
#define WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE 0x00001
#define WUFFS_BASE__TOKEN__VBD__STRING__ALL_ASCII 0x00010
// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between
// VBD_FILLER, VBD_STRING and VBD_BYTES.
#define WUFFS_BASE__TOKEN__VBD__BYTES__INCOMPLETE 0x00001
// "D_DST_S_SRC" means that it takes S source bytes (possibly padded) to
// produce D destination bytes. For example,
// WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_4_SRC_BACKSLASH_X means that the source
// looks like "\\x23\\x67\\xAB", where 12 src bytes encode 3 dst bytes.
#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_1_SRC_RAW 0x00010
#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_2_SRC_HEX 0x00020
#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_4_SRC_BACKSLASH_X 0x00040
#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_6_SRC_BACKSLASH_U 0x00080
#define WUFFS_BASE__TOKEN__VBD__BYTES__3_DST_4_SRC_BASE_64_STD 0x00100
#define WUFFS_BASE__TOKEN__VBD__BYTES__3_DST_4_SRC_BASE_64_URL 0x00200
#define WUFFS_BASE__TOKEN__VBD__BYTES__4_DST_5_SRC_ASCII_85 0x00400
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL 0x00001
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__UNDEFINED 0x00101
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__NULL 0x00201
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__FALSE 0x00401
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__TRUE 0x00801
// For a source string of "123" or "0x9A", it is valid for a tokenizer to
// return any one of:
// - WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT.
// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED.
// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED.
//
// For a source string of "+123" or "-0x9A", only the first two are valid.
//
// For a source string of "123.", only the first one is valid.
#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00002
#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED 0x00004
#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED 0x00008
#define WUFFS_BASE__TOKEN__VBD__UNICODE_CODE_POINT__MAX_INCL 0x10FFFF
static inline uint64_t //
wuffs_base__token__value(const wuffs_base__token* t) {
return (t->repr >> WUFFS_BASE__TOKEN__VALUE__SHIFT) &
@@ -18777,7 +18826,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
goto label__0__continue;
}
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(6291457)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(2097153)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__0__break;
}
@@ -18795,7 +18844,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) {
if (v_string_length > 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(6291457))
(((uint64_t)(2097153))
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(((uint64_t)(v_string_length))))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
@@ -18816,7 +18865,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
if (v_c <= 34) {
if (v_c == 34) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(6291456))
(((uint64_t)(2097152))
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(((uint64_t)(v_string_length))))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
@@ -18825,7 +18874,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
} else if (v_c < 32) {
if (v_string_length > 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(6291457))
(((uint64_t)(2097153))
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(((uint64_t)(v_string_length))))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
@@ -18838,7 +18887,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
} else if (v_c == 92) {
if (v_string_length > 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(6291457))
(((uint64_t)(2097153))
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(((uint64_t)(v_string_length))))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
@@ -18865,7 +18914,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
if (v_c > 0) {
(iop_a_src += 2, wuffs_base__make_empty_struct());
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)((8388608 | ((uint64_t)((v_c & 127))))))
(((uint64_t)((10485760 | ((uint64_t)((v_c & 127))))))
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__string_loop__continue;
@@ -18906,7 +18955,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
} else if ((v_uni4_value < 55296) || (57343 < v_uni4_value)) {
(iop_a_src += 6, wuffs_base__make_empty_struct());
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)((8388608 | ((uint64_t)(v_uni4_value)))))
(((uint64_t)((10485760 | ((uint64_t)(v_uni4_value)))))
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(6)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__string_loop__continue;
@@ -18961,8 +19010,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
(iop_a_src += 8, wuffs_base__make_empty_struct());
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(
(8388608 | ((uint64_t)((v_uni4_high_surrogate +
v_uni4_value))))))
(10485760 | ((uint64_t)((v_uni4_high_surrogate +
v_uni4_value))))))
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(12))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
@@ -18988,7 +19037,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
(iop_a_src += 1, wuffs_base__make_empty_struct());
if (v_string_length >= 65534) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(6291457)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(2097153)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(65535)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
goto label__string_loop__continue;
@@ -19055,9 +19104,9 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
iop_a_src = a_src->data.ptr + a_src->meta.ri;
}
v_number_status = (v_number_length >> 8);
v_token_value = 4194306;
v_token_value = 8388612;
if ((v_number_length & 128) != 0) {
v_token_value = 4194308;
v_token_value = 8388610;
}
v_number_length = (v_number_length & 127);
if (v_number_status == 0) {
@@ -19099,12 +19148,12 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
label__2__break:;
goto label__goto_parsed_a_leaf_value__break;
} else if (v_class == 5) {
v_token_value = 2113553;
v_token_value = 6307857;
if (v_depth == 0) {
} else if (0 != (v_expect_after_value & 64)) {
v_token_value = 2113601;
v_token_value = 6307905;
} else {
v_token_value = 2113569;
v_token_value = 6307873;
}
if (v_depth >= 1024) {
status = wuffs_base__make_status(
@@ -19127,7 +19176,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
(iop_a_src += 1, wuffs_base__make_empty_struct());
if (v_depth <= 1) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(2101314)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(6295618)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__outer__break;
}
@@ -19137,25 +19186,25 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
if (0 == (self->private_data.f_stack[v_stack_byte] &
(((uint32_t)(1)) << v_stack_bit))) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(2105410)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(6299714)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_expect = 260;
v_expect_after_value = 260;
} else {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(2113602)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(6307906)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_expect = 68;
v_expect_after_value = 68;
}
goto label__outer__continue;
} else if (v_class == 7) {
v_token_value = 2105361;
v_token_value = 6299665;
if (v_depth == 0) {
} else if (0 != (v_expect_after_value & 64)) {
v_token_value = 2105409;
v_token_value = 6299713;
} else {
v_token_value = 2105377;
v_token_value = 6299681;
}
if (v_depth >= 1024) {
status = wuffs_base__make_status(
@@ -19178,7 +19227,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
(iop_a_src += 1, wuffs_base__make_empty_struct());
if (v_depth <= 1) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(2101282)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(6295586)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__outer__break;
}
@@ -19188,13 +19237,13 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
if (0 == (self->private_data.f_stack[v_stack_byte] &
(((uint32_t)(1)) << v_stack_bit))) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(2105378)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(6299682)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_expect = 260;
v_expect_after_value = 260;
} else {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(2113570)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(6307874)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_expect = 68;
v_expect_after_value = 68;
@@ -19205,7 +19254,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
111546413966853);
if (v_match == 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194369)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(8389633)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(5)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
if (((uint64_t)(io2_a_src - iop_a_src)) < 5) {
status = wuffs_base__make_status(
@@ -19225,7 +19274,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
435762131972);
if (v_match == 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194433)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(8390657)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(4)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
if (((uint64_t)(io2_a_src - iop_a_src)) < 4) {
status = wuffs_base__make_status(
@@ -19245,7 +19294,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
465676103172);
if (v_match == 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(8389121)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(4)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
if (((uint64_t)(io2_a_src - iop_a_src)) < 4) {
status = wuffs_base__make_status(
+26 -26
View File
@@ -174,7 +174,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
continue
}
args.dst.write_fast_token!(
value: 0x60_0001,
value: 0x20_0001,
length: 0)
break
}
@@ -192,7 +192,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
if args.src.available() <= 0 {
if string_length > 0 {
args.dst.write_fast_token!(
value: 0x60_0001,
value: 0x20_0001,
length: string_length as base.u64)
string_length = 0
}
@@ -208,7 +208,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
if c <= 0x22 { // 0x22 is '"'.
if c == 0x22 { // 0x22 is '"'.
args.dst.write_fast_token!(
value: 0x60_0000,
value: 0x20_0000,
length: string_length as base.u64)
string_length = 0
break.string_loop
@@ -216,7 +216,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
} else if c < 0x20 { // 0x20 is ' '.
if string_length > 0 {
args.dst.write_fast_token!(
value: 0x60_0001,
value: 0x20_0001,
length: string_length as base.u64)
string_length = 0
}
@@ -226,7 +226,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
} else if c == 0x5C { // 0x5C is '\\'.
if string_length > 0 {
args.dst.write_fast_token!(
value: 0x60_0001,
value: 0x20_0001,
length: string_length as base.u64)
string_length = 0
if args.dst.available() <= 0 {
@@ -248,7 +248,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
if c > 0 {
args.src.skip32_fast!(actual: 2, worst_case: 2)
args.dst.write_fast_token!(
value: 0x80_0000 | ((c & 0x7F) as base.u64),
value: 0xA0_0000 | ((c & 0x7F) as base.u64),
length: 2)
continue.string_loop
@@ -288,7 +288,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
// Not a Unicode surrogate. We're good.
args.src.skip32_fast!(actual: 6, worst_case: 6)
args.dst.write_fast_token!(
value: 0x80_0000 | (uni4_value as base.u64),
value: 0xA0_0000 | (uni4_value as base.u64),
length: 6)
continue.string_loop
@@ -351,7 +351,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
uni4_value -= 0xDC00
args.src.skip32_fast!(actual: 8, worst_case: 8)
args.dst.write_fast_token!(
value: 0x80_0000 | ((uni4_high_surrogate + uni4_value) as base.u64),
value: 0xA0_0000 | ((uni4_high_surrogate + uni4_value) as base.u64),
length: 12)
continue.string_loop
@@ -378,7 +378,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
args.src.skip32_fast!(actual: 1, worst_case: 1)
if string_length >= 0xFFFE {
args.dst.write_fast_token!(
value: 0x60_0001,
value: 0x20_0001,
length: 0xFFFF)
string_length = 0
continue.string_loop
@@ -448,9 +448,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
{
number_length = this.decode_number!(src: args.src)
number_status = number_length >> 8
token_value = 0x40_0002
token_value = 0x80_0004
if (number_length & 0x80) <> 0 {
token_value = 0x40_0004
token_value = 0x80_0002
}
number_length = number_length & 0x7F
if number_status == 0 {
@@ -486,13 +486,13 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
// -------- END parse numbers.
} else if class == 0x05 { // 0x05 is CLASS_OPEN_CURLY_BRACE.
token_value = 0x20_4011
token_value = 0x60_4011
if depth == 0 {
// No-op.
} else if 0 <> (expect_after_value & 0x0040) { // 0x0040 is EXPECT_CLOSE_CURLY_BRACE.
token_value = 0x20_4041
token_value = 0x60_4041
} else {
token_value = 0x20_4021
token_value = 0x60_4021
}
if depth >= 1024 {
return "#unsupported recursion depth"
@@ -514,7 +514,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
args.src.skip32_fast!(actual: 1, worst_case: 1)
if depth <= 1 {
args.dst.write_fast_token!(
value: 0x20_1042,
value: 0x60_1042,
length: 1)
break.outer
}
@@ -523,14 +523,14 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
stack_bit = (depth - 1) & 31
if 0 == (this.stack[stack_byte] & ((1 as base.u32) << stack_bit)) {
args.dst.write_fast_token!(
value: 0x20_2042,
value: 0x60_2042,
length: 1)
// 0x0104 is (EXPECT_SQUARE_CLOSE_BRACKET | EXPECT_COMMA).
expect = 0x0104
expect_after_value = 0x0104
} else {
args.dst.write_fast_token!(
value: 0x20_4042,
value: 0x60_4042,
length: 1)
// 0x0044 is (EXPECT_CURLY_CLOSE_BRACE | EXPECT_COMMA).
expect = 0x0044
@@ -539,13 +539,13 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
continue.outer
} else if class == 0x07 { // 0x07 is CLASS_OPEN_SQUARE_BRACKET.
token_value = 0x20_2011
token_value = 0x60_2011
if depth == 0 {
// No-op.
} else if 0 <> (expect_after_value & 0x0040) { // 0x0040 is EXPECT_CLOSE_CURLY_BRACE.
token_value = 0x20_2041
token_value = 0x60_2041
} else {
token_value = 0x20_2021
token_value = 0x60_2021
}
if depth >= 1024 {
return "#unsupported recursion depth"
@@ -567,7 +567,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
args.src.skip32_fast!(actual: 1, worst_case: 1)
if depth <= 1 {
args.dst.write_fast_token!(
value: 0x20_1022,
value: 0x60_1022,
length: 1)
break.outer
}
@@ -576,14 +576,14 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
stack_bit = (depth - 1) & 31
if 0 == (this.stack[stack_byte] & ((1 as base.u32) << stack_bit)) {
args.dst.write_fast_token!(
value: 0x20_2022,
value: 0x60_2022,
length: 1)
// 0x0104 is (EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA).
expect = 0x0104
expect_after_value = 0x0104
} else {
args.dst.write_fast_token!(
value: 0x20_4022,
value: 0x60_4022,
length: 1)
// 0x0044 is (EXPECT_CLOSE_CURLY_BRACE | EXPECT_COMMA).
expect = 0x0044
@@ -595,7 +595,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
match = args.src.match7(a: 0x6573_6C61_6605) // 5 bytes "false".
if match == 0 {
args.dst.write_fast_token!(
value: 0x40_0041,
value: 0x80_0401,
length: 5)
if args.src.available() < 5 {
return "#internal error: inconsistent I/O"
@@ -611,7 +611,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
match = args.src.match7(a: 0x65_7572_7404) // 4 bytes "true".
if match == 0 {
args.dst.write_fast_token!(
value: 0x40_0081,
value: 0x80_0801,
length: 4)
if args.src.available() < 4 {
return "#internal error: inconsistent I/O"
@@ -627,7 +627,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
match = args.src.match7(a: 0x6C_6C75_6E04) // 4 bytes "null".
if match == 0 {
args.dst.write_fast_token!(
value: 0x40_0021,
value: 0x80_0201,
length: 4)
if args.src.available() < 4 {
return "#internal error: inconsistent I/O"