mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-20 08:47:28 +01:00
Have tokens include explicit link bits
This commit is contained in:
1 parent
cb37a56997
commit
d1c928a1ed
11 files changed
+533
-388
No files matched your search
+16
-21
@@ -99,8 +99,6 @@ wuffs_base__token_buffer tok;
|
||||
// token. An invariant is that (curr_token_end_src_index <= src.meta.ri).
|
||||
size_t curr_token_end_src_index;
|
||||
|
||||
bool prev_token_incomplete;
|
||||
|
||||
uint64_t depth;
|
||||
|
||||
enum class context {
|
||||
@@ -132,8 +130,6 @@ initialize_globals(int argc, char** argv) {
|
||||
|
||||
curr_token_end_src_index = 0;
|
||||
|
||||
prev_token_incomplete = false;
|
||||
|
||||
depth = 0;
|
||||
|
||||
ctx = context::none;
|
||||
@@ -319,19 +315,20 @@ handle_token(wuffs_base__token t) {
|
||||
goto after_value;
|
||||
}
|
||||
|
||||
// Write preceding whitespace and punctuation, if it wasn't ']' or '}'.
|
||||
if (!prev_token_incomplete) {
|
||||
if (ctx == context::in_dict_after_key) {
|
||||
TRY(write_dst(": ", 2));
|
||||
} else if (ctx != context::none) {
|
||||
if ((ctx != context::in_list_after_bracket) &&
|
||||
(ctx != context::in_dict_after_brace)) {
|
||||
TRY(write_dst(",", 1));
|
||||
}
|
||||
TRY(write_dst("\n", 1));
|
||||
for (size_t i = 0; i < depth; i++) {
|
||||
TRY(write_dst(INDENT_STRING, indent));
|
||||
}
|
||||
// Write preceding whitespace and punctuation, if it wasn't ']', '}' or a
|
||||
// continuation of a multi-token chain.
|
||||
if (t.link_prev()) {
|
||||
// No-op.
|
||||
} else if (ctx == context::in_dict_after_key) {
|
||||
TRY(write_dst(": ", 2));
|
||||
} else if (ctx != context::none) {
|
||||
if ((ctx != context::in_list_after_bracket) &&
|
||||
(ctx != context::in_dict_after_brace)) {
|
||||
TRY(write_dst(",", 1));
|
||||
}
|
||||
TRY(write_dst("\n", 1));
|
||||
for (size_t i = 0; i < depth; i++) {
|
||||
TRY(write_dst(INDENT_STRING, indent));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -339,7 +336,7 @@ handle_token(wuffs_base__token t) {
|
||||
// value (number, string or literal).
|
||||
switch (vbc) {
|
||||
case WUFFS_BASE__TOKEN__VBC__STRING:
|
||||
if (!prev_token_incomplete) {
|
||||
if (!t.link_prev()) {
|
||||
TRY(write_dst("\"", 1));
|
||||
}
|
||||
|
||||
@@ -352,9 +349,7 @@ handle_token(wuffs_base__token t) {
|
||||
return "main: internal error: unexpected string-token conversion";
|
||||
}
|
||||
|
||||
prev_token_incomplete =
|
||||
vbd & WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE;
|
||||
if (prev_token_incomplete) {
|
||||
if (t.link_next()) {
|
||||
return nullptr;
|
||||
}
|
||||
TRY(write_dst("\"", 1));
|
||||
|
||||
@@ -17,22 +17,52 @@
|
||||
// ---------------- Tokens
|
||||
|
||||
typedef struct {
|
||||
// The repr is divided as:
|
||||
// - Bits 63 .. 40 (24 bits) is the major value.
|
||||
// - Bits 39 .. 16 (24 bits) is the minor value.
|
||||
// The repr's 64 bits are divided as:
|
||||
//
|
||||
// +-----+-------------+-------+-------------+-----+-----+-----------+
|
||||
// | 1 | 21 | 3 | 21 | 1 | 1 | 16 |
|
||||
// +-----+-------------+-------+-------------+-----+-----+-----------+
|
||||
// [..................value..................] LP LN length
|
||||
// [..0..|.value_major.|.....value_minor.....]
|
||||
// [..0..|.........VBC.........|.....VBD.....]
|
||||
//
|
||||
// The broad divisions are:
|
||||
// - Bits 63 .. 18 (46 bits) is the value.
|
||||
// - Bits 17 .. 16 ( 2 bits) is LP and LN (link_prev and link_next).
|
||||
// - Bits 15 .. 0 (16 bits) is the length.
|
||||
//
|
||||
// The major value is a [Base38](doc/note/base38-and-fourcc.md) value. If
|
||||
// zero (special cased for Wuffs' built-in "base" package) then the minor
|
||||
// value is further sub-divided:
|
||||
// - Bits 39 .. 37 ( 3 bits) is the value_base_category.
|
||||
// - Bits 36 .. 16 (21 bits) is the value_base_detail.
|
||||
// The value bits can be sub-divided in multiple ways:
|
||||
// - Bits 63 .. 63 ( 1 bits) is reserved (a zero bit).
|
||||
// - Bits 62 .. 42 (21 bits) is the value_major.
|
||||
// - Bits 41 .. 18 (24 bits) is the value_minor.
|
||||
// - Bits 62 .. 39 (24 bits) is the VBC (value_base_category).
|
||||
// - Bits 38 .. 18 (21 bits) is the VBD (value_base_detail).
|
||||
//
|
||||
// In particular, at 21 bits, the value_base_detail can hold every valid
|
||||
// Unicode code point.
|
||||
// The value_major is a 21-bit [Base38](doc/note/base38-and-fourcc.md) value.
|
||||
// If all of its bits are zero (special cased for Wuffs' built-in "base"
|
||||
// package) then the value_minor is further sub-divided:
|
||||
// - Bits 41 .. 39 ( 3 bits) is the VBC (value_base_category).
|
||||
// - Bits 38 .. 18 (21 bits) is the VBD (value_base_detail).
|
||||
//
|
||||
// If the major value is non-zero then the minor value has whatever arbitrary
|
||||
// meaning the tokenizer's package assigns to it.
|
||||
// The high 46 bits (bits 63 .. 18) only have VBC and VBD semantics when the
|
||||
// high 22 bits (the value_major) are all zero. An equivalent test is that
|
||||
// the high 25 bits (the notional VBC) has a numerical value less than 8.
|
||||
//
|
||||
// At 21 bits, the VBD can hold every valid Unicode code point.
|
||||
//
|
||||
// If value_major is non-zero then value_minor has whatever arbitrary meaning
|
||||
// the tokenizer's package assigns to it.
|
||||
//
|
||||
// Multiple consecutive tokens can form a larger conceptual unit. For
|
||||
// example, an "abc\tz" string is a single higher level concept but at the
|
||||
// lower level, it could consist of multiple tokens: the quotes '"', the
|
||||
// ASCII texts "abc" and "z" and the backslash-escaped tab '\t'. The LP and
|
||||
// LN (link_prev and link_next) bits denote tokens that are part of a
|
||||
// multi-token chain:
|
||||
// - LP means that this token is not the first (there is a previous token).
|
||||
// - LN means that this token is not the last (there is a next token).
|
||||
//
|
||||
// In particular, a stand-alone token will have both link bits set to zero.
|
||||
uint64_t repr;
|
||||
|
||||
#ifdef __cplusplus
|
||||
@@ -41,6 +71,8 @@ typedef struct {
|
||||
inline uint64_t value_minor() const;
|
||||
inline uint64_t value_base_category() const;
|
||||
inline uint64_t value_base_detail() const;
|
||||
inline bool link_prev() const;
|
||||
inline bool link_next() const;
|
||||
inline uint64_t length() const;
|
||||
#endif // __cplusplus
|
||||
|
||||
@@ -55,20 +87,25 @@ wuffs_base__make_token(uint64_t repr) {
|
||||
|
||||
// --------
|
||||
|
||||
#define WUFFS_BASE__TOKEN__VALUE__MASK 0xFFFFFFFFFFFF
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0xFFFFFF
|
||||
#define WUFFS_BASE__TOKEN__VALUE__MASK 0x3FFFFFFFFFFF
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0x3FFFFF
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF
|
||||
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x7FFFFFF
|
||||
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x1FFFFFF
|
||||
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF
|
||||
#define WUFFS_BASE__TOKEN__LINK__MASK 0x3
|
||||
#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF
|
||||
|
||||
#define WUFFS_BASE__TOKEN__VALUE__SHIFT 16
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 40
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 16
|
||||
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 37
|
||||
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16
|
||||
#define WUFFS_BASE__TOKEN__VALUE__SHIFT 18
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 18
|
||||
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 39
|
||||
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 18
|
||||
#define WUFFS_BASE__TOKEN__LINK__SHIFT 16
|
||||
#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0
|
||||
|
||||
#define WUFFS_BASE__TOKEN__LINK_PREV 0x20000
|
||||
#define WUFFS_BASE__TOKEN__LINK_NEXT 0x10000
|
||||
|
||||
// --------
|
||||
|
||||
#define WUFFS_BASE__TOKEN__VBC__FILLER 0
|
||||
@@ -77,41 +114,19 @@ wuffs_base__make_token(uint64_t repr) {
|
||||
#define WUFFS_BASE__TOKEN__VBC__NUMBER 3
|
||||
#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 4
|
||||
|
||||
// --------
|
||||
// --------
|
||||
|
||||
// INCOMPLETE means that this token combines with the following token.
|
||||
//
|
||||
// For example, tokenizing a comment that is longer than the source buffer can
|
||||
// result in multiple VBC__FILLER tokens. All but the last one is INCOMPLETE.
|
||||
//
|
||||
// By convention, whitespace is not marked incomplete. Two whitespace tokens of
|
||||
// length 30 are equivalent to one whitespace token of length 60.
|
||||
//
|
||||
// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between
|
||||
// VBD_FILLER and VBD_STRING.
|
||||
#define WUFFS_BASE__TOKEN__VBD__FILLER__INCOMPLETE 0x00001
|
||||
|
||||
#define WUFFS_BASE__TOKEN__VBD__FILLER__END_OF_CONSECUTIVE_COMMENTS 0x00010
|
||||
#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00020
|
||||
#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00040
|
||||
#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00001
|
||||
#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00002
|
||||
|
||||
// --------
|
||||
|
||||
// INCOMPLETE means that this token combines with the following token.
|
||||
//
|
||||
// For example, tokenizing a string that is longer than the source buffer can
|
||||
// result in multiple VBC__STRING tokens. All but the last one is INCOMPLETE.
|
||||
//
|
||||
// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between
|
||||
// VBD_FILLER and VBD_STRING.
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE 0x00001
|
||||
|
||||
// "DEFINITELY_FOO" means that the source bytes (and also the destination
|
||||
// bytes, assuming 1_DST_1_SRC_COPY) are in the FOO format. Definitely means
|
||||
// that the lack of the bit is conservative: it is valid for all-ASCII strings
|
||||
// to have neither DEFINITELY_ASCII or DEFINITELY_UTF_8 bits set.
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00010
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00020
|
||||
// "DEFINITELY_FOO" means that the destination bytes (and also the source
|
||||
// bytes, for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that
|
||||
// the lack of the bit is conservative: it is valid for all-ASCII strings to
|
||||
// have neither DEFINITELY_UTF_8 or DEFINITELY_ASCII bits set.
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00001
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00002
|
||||
|
||||
// "CONVERT_D_DST_S_SRC" means that multiples of S source bytes (possibly
|
||||
// padded) produces multiples of D destination bytes. For example,
|
||||
@@ -121,21 +136,13 @@ wuffs_base__make_token(uint64_t repr) {
|
||||
// When src is the empty string, multiple conversion algorithms are applicable
|
||||
// (so these bits are not necessarily mutually exclusive), all producing the
|
||||
// same empty dst string.
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00100
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00200
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00400
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00800
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x01000
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x02000
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x04000
|
||||
|
||||
// --------
|
||||
|
||||
// UNICODE_CODE_POINT tokens have no room in their VBD for an INCOMPLETE bit.
|
||||
// All 21 bits are used to hold the Unicode code point. Such tokens preserve
|
||||
// the INCOMPLETEness of the previous token (if a FILLER or STRING).
|
||||
|
||||
#define WUFFS_BASE__TOKEN__VBD__UNICODE_CODE_POINT__MAX_INCL 0x10FFFF
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00010
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00020
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00040
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00080
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x00100
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x00200
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x00400
|
||||
|
||||
// --------
|
||||
|
||||
@@ -201,6 +208,16 @@ wuffs_base__token__value_base_detail(const wuffs_base__token* t) {
|
||||
WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK;
|
||||
}
|
||||
|
||||
static inline bool //
|
||||
wuffs_base__token__link_prev(const wuffs_base__token* t) {
|
||||
return t->repr & WUFFS_BASE__TOKEN__LINK_PREV;
|
||||
}
|
||||
|
||||
static inline bool //
|
||||
wuffs_base__token__link_next(const wuffs_base__token* t) {
|
||||
return t->repr & WUFFS_BASE__TOKEN__LINK_NEXT;
|
||||
}
|
||||
|
||||
static inline uint64_t //
|
||||
wuffs_base__token__length(const wuffs_base__token* t) {
|
||||
return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) &
|
||||
@@ -234,6 +251,16 @@ wuffs_base__token::value_base_detail() const {
|
||||
return wuffs_base__token__value_base_detail(this);
|
||||
}
|
||||
|
||||
inline bool //
|
||||
wuffs_base__token::link_prev() const {
|
||||
return wuffs_base__token__link_prev(this);
|
||||
}
|
||||
|
||||
inline bool //
|
||||
wuffs_base__token::link_next() const {
|
||||
return wuffs_base__token__link_next(this);
|
||||
}
|
||||
|
||||
inline uint64_t //
|
||||
wuffs_base__token::length() const {
|
||||
return wuffs_base__token__length(this);
|
||||
|
||||
@@ -325,11 +325,27 @@ func (g *gen) writeBuiltinIOWriter(b *buffer, recv *a.Expr, method t.ID, args []
|
||||
func (g *gen) writeBuiltinTokenWriter(b *buffer, recv *a.Expr, method t.ID, args []*a.Node, depth uint32) error {
|
||||
if method == t.IDWriteFastToken {
|
||||
b.printf("*iop_a_dst++ = wuffs_base__make_token((((uint64_t)(")
|
||||
if err := g.writeExpr(b, args[0].AsArg().Value(), depth); err != nil {
|
||||
|
||||
if cv := args[0].AsArg().Value().ConstValue(); (cv == nil) || (cv.Sign() != 0) {
|
||||
if err := g.writeExpr(b, args[0].AsArg().Value(), depth); err != nil {
|
||||
return err
|
||||
}
|
||||
b.writes(")) << WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT) | (((uint64_t)(")
|
||||
}
|
||||
|
||||
if err := g.writeExpr(b, args[1].AsArg().Value(), depth); err != nil {
|
||||
return err
|
||||
}
|
||||
b.writes(")) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(")
|
||||
if err := g.writeExpr(b, args[1].AsArg().Value(), depth); err != nil {
|
||||
b.writes(")) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(")
|
||||
|
||||
if cv := args[2].AsArg().Value().ConstValue(); (cv == nil) || (cv.Sign() != 0) {
|
||||
if err := g.writeExpr(b, args[2].AsArg().Value(), depth); err != nil {
|
||||
return err
|
||||
}
|
||||
b.writes(")) << WUFFS_BASE__TOKEN__LINK__SHIFT) | (((uint64_t)(")
|
||||
}
|
||||
|
||||
if err := g.writeExpr(b, args[3].AsArg().Value(), depth); err != nil {
|
||||
return err
|
||||
}
|
||||
b.writes(")) << WUFFS_BASE__TOKEN__LENGTH__SHIFT))")
|
||||
@@ -699,7 +715,7 @@ func (g *gen) writeBuiltinQuestionCall(b *buffer, n *a.Expr, depth uint32) error
|
||||
if err := g.writeExpr(b, n.Args()[0].AsArg().Value(), depth); err != nil {
|
||||
return err
|
||||
}
|
||||
b.writes(")) << WUFFS_BASE__TOKEN__VALUE__SHIFT) | (((uint64_t)(")
|
||||
b.writes(")) << WUFFS_BASE__TOKEN__VLL__SHIFT) | (((uint64_t)(")
|
||||
if err := g.writeExpr(b, n.Args()[1].AsArg().Value(), depth); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+10
-10
@@ -308,26 +308,26 @@ const baseTokenPrivateH = "" +
|
||||
""
|
||||
|
||||
const baseTokenPublicH = "" +
|
||||
"// ---------------- Tokens\n\ntypedef struct {\n // The repr is divided as:\n // - Bits 63 .. 40 (24 bits) is the major value.\n // - Bits 39 .. 16 (24 bits) is the minor value.\n // - Bits 15 .. 0 (16 bits) is the length.\n //\n // The major value is a [Base38](doc/note/base38-and-fourcc.md) value. If\n // zero (special cased for Wuffs' built-in \"base\" package) then the minor\n // value is further sub-divided:\n // - Bits 39 .. 37 ( 3 bits) is the value_base_category.\n // - Bits 36 .. 16 (21 bits) is the value_base_detail.\n //\n // In particular, at 21 bits, the value_base_detail can hold every valid\n // Unicode code point.\n //\n // If the major value is non-zero then the minor value has whatever arbitrary\n // meaning the tokenizer's package assigns to it.\n uint64_t repr;\n\n#ifdef __cplusplus\n inline uint64_t value() const;\n inline uint64_t value_major() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_category() const;\n inline uint64_t value_base_detail() const;\n inlin" +
|
||||
"e uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\n " +
|
||||
"// ---------------- Tokens\n\ntypedef struct {\n // The repr's 64 bits are divided as:\n //\n // +-----+-------------+-------+-------------+-----+-----+-----------+\n // | 1 | 21 | 3 | 21 | 1 | 1 | 16 |\n // +-----+-------------+-------+-------------+-----+-----+-----------+\n // [..................value..................] LP LN length\n // [..0..|.value_major.|.....value_minor.....]\n // [..0..|.........VBC.........|.....VBD.....]\n //\n // The broad divisions are:\n // - Bits 63 .. 18 (46 bits) is the value.\n // - Bits 17 .. 16 ( 2 bits) is LP and LN (link_prev and link_next).\n // - Bits 15 .. 0 (16 bits) is the length.\n //\n // The value bits can be sub-divided in multiple ways:\n // - Bits 63 .. 63 ( 1 bits) is reserved (a zero bit).\n // - Bits 62 .. 42 (21 bits) is the value_major.\n // - Bits 41 .. 18 (24 bits) is the value_minor.\n // - Bits 62 .. 39 (24 bits) is the VBC (value_base_category).\n // - Bits 38 .. 18 (21 bits) is the VBD (value_base_de" +
|
||||
"tail).\n //\n // The value_major is a 21-bit [Base38](doc/note/base38-and-fourcc.md) value.\n // If all of its bits are zero (special cased for Wuffs' built-in \"base\"\n // package) then the value_minor is further sub-divided:\n // - Bits 41 .. 39 ( 3 bits) is the VBC (value_base_category).\n // - Bits 38 .. 18 (21 bits) is the VBD (value_base_detail).\n //\n // The high 46 bits (bits 63 .. 18) only have VBC and VBD semantics when the\n // high 22 bits (the value_major) are all zero. An equivalent test is that\n // the high 25 bits (the notional VBC) has a numerical value less than 8.\n //\n // At 21 bits, the VBD can hold every valid Unicode code point.\n //\n // If value_major is non-zero then value_minor has whatever arbitrary meaning\n // the tokenizer's package assigns to it.\n //\n // Multiple consecutive tokens can form a larger conceptual unit. For\n // example, an \"abc\\tz\" string is a single higher level concept but at the\n // lower level, it could consist of multiple tokens: the quotes '\"', the\n " +
|
||||
"// ASCII texts \"abc\" and \"z\" and the backslash-escaped tab '\\t'. The LP and\n // LN (link_prev and link_next) bits denote tokens that are part of a\n // multi-token chain:\n // - LP means that this token is not the first (there is a previous token).\n // - LN means that this token is not the last (there is a next token).\n //\n // In particular, a stand-alone token will have both link bits set to zero.\n uint64_t repr;\n\n#ifdef __cplusplus\n inline uint64_t value() const;\n inline uint64_t value_major() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_category() const;\n inline uint64_t value_base_detail() const;\n inline bool link_prev() const;\n inline bool link_next() const;\n inline uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\n " +
|
||||
"" +
|
||||
"// --------\n\n#define WUFFS_BASE__TOKEN__VALUE__MASK 0xFFFFFFFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x7FFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF\n#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 16\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 40\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 16\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 37\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n " +
|
||||
"// --------\n\n#define WUFFS_BASE__TOKEN__VALUE__MASK 0x3FFFFFFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0x3FFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x1FFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF\n#define WUFFS_BASE__TOKEN__LINK__MASK 0x3\n#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 39\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 18\n#define WUFFS_BASE__TOKEN__LINK__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n#define WUFFS_BASE__TOKEN__LINK_PREV 0x20000\n#define WUFFS_BASE__TOKEN__LINK_NEXT 0x10000\n\n " +
|
||||
"" +
|
||||
"// --------\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRING 1\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 2\n#define WUFFS_BASE__TOKEN__VBC__NUMBER 3\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 4\n\n" +
|
||||
"// --------\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRING 1\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 2\n#define WUFFS_BASE__TOKEN__VBC__NUMBER 3\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 4\n\n " +
|
||||
"" +
|
||||
"// --------\n\n// INCOMPLETE means that this token combines with the following token.\n//\n// For example, tokenizing a comment that is longer than the source buffer can\n// result in multiple VBC__FILLER tokens. All but the last one is INCOMPLETE.\n//\n// By convention, whitespace is not marked incomplete. Two whitespace tokens of\n// length 30 are equivalent to one whitespace token of length 60.\n//\n// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between\n// VBD_FILLER and VBD_STRING.\n#define WUFFS_BASE__TOKEN__VBD__FILLER__INCOMPLETE 0x00001\n\n#define WUFFS_BASE__TOKEN__VBD__FILLER__END_OF_CONSECUTIVE_COMMENTS 0x00010\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00020\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00040\n\n" +
|
||||
"// --------\n\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00001\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00002\n\n" +
|
||||
"" +
|
||||
"// --------\n\n// INCOMPLETE means that this token combines with the following token.\n//\n// For example, tokenizing a string that is longer than the source buffer can\n// result in multiple VBC__STRING tokens. All but the last one is INCOMPLETE.\n//\n// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between\n// VBD_FILLER and VBD_STRING.\n#define WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE 0x00001\n\n// \"DEFINITELY_FOO\" means that the source bytes (and also the destination\n// bytes, assuming 1_DST_1_SRC_COPY) are in the FOO format. Definitely means\n// that the lack of the bit is conservative: it is valid for all-ASCII strings\n// to have neither DEFINITELY_ASCII or DEFINITELY_UTF_8 bits set.\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00020\n\n// \"CONVERT_D_DST_S_SRC\" means that multiples of S source bytes (possibly\n// padded) produces multiples of D destination bytes. For example,\n// CONVERT_1_DST_4_SRC_BACKSLASH_X means a sourc" +
|
||||
"e like \"\\\\x23\\\\x67\\\\xAB\", where\n// 12 src bytes encode 3 dst bytes.\n//\n// When src is the empty string, multiple conversion algorithms are applicable\n// (so these bits are not necessarily mutually exclusive), all producing the\n// same empty dst string.\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00100\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00200\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00400\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00800\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x01000\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x02000\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x04000\n\n " +
|
||||
"" +
|
||||
"// --------\n\n // UNICODE_CODE_POINT tokens have no room in their VBD for an INCOMPLETE bit.\n // All 21 bits are used to hold the Unicode code point. Such tokens preserve\n // the INCOMPLETEness of the previous token (if a FILLER or STRING).\n\n#define WUFFS_BASE__TOKEN__VBD__UNICODE_CODE_POINT__MAX_INCL 0x10FFFF\n\n " +
|
||||
"// --------\n\n// \"DEFINITELY_FOO\" means that the destination bytes (and also the source\n// bytes, for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that\n// the lack of the bit is conservative: it is valid for all-ASCII strings to\n// have neither DEFINITELY_UTF_8 or DEFINITELY_ASCII bits set.\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00002\n\n// \"CONVERT_D_DST_S_SRC\" means that multiples of S source bytes (possibly\n// padded) produces multiples of D destination bytes. For example,\n// CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like \"\\\\x23\\\\x67\\\\xAB\", where\n// 12 src bytes encode 3 dst bytes.\n//\n// When src is the empty string, multiple conversion algorithms are applicable\n// (so these bits are not necessarily mutually exclusive), all producing the\n// same empty dst string.\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00020\n#" +
|
||||
"define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00080\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x00100\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x00200\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x00400\n\n " +
|
||||
"" +
|
||||
"// --------\n\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL 0x00001\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__UNDEFINED 0x00101\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__NULL 0x00201\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__FALSE 0x00401\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__TRUE 0x00801\n\n// For a source string of \"123\" or \"0x9A\", it is valid for a tokenizer to\n// return any one of:\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT.\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED.\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED.\n//\n// For a source string of \"+123\" or \"-0x9A\", only the first two are valid.\n//\n// For a source string of \"123.\", only the first one is valid.\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00002\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED 0x00004\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED 0x00008\n\n " +
|
||||
"" +
|
||||
"// --------\n\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000\n\n" +
|
||||
"" +
|
||||
"// --------\n\nstatic inline uint64_t //\nwuffs_base__token__value(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_major(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_minor(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MINOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_category(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_detail(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_BASE_DETA" +
|
||||
"IL__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__length(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) &\n WUFFS_BASE__TOKEN__LENGTH__MASK;\n}\n\n#ifdef __cplusplus\n\ninline uint64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_major() const {\n return wuffs_base__token__value_major(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__token__value_base_detail(this);\n}\n\ninline uint64_t //\nwuffs_base__token::length() const {\n return wuffs_base__token__length(this);\n}\n\n#endif // __cplusplus\n\n" +
|
||||
"IL__MASK;\n}\n\nstatic inline bool //\nwuffs_base__token__link_prev(const wuffs_base__token* t) {\n return t->repr & WUFFS_BASE__TOKEN__LINK_PREV;\n}\n\nstatic inline bool //\nwuffs_base__token__link_next(const wuffs_base__token* t) {\n return t->repr & WUFFS_BASE__TOKEN__LINK_NEXT;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__length(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) &\n WUFFS_BASE__TOKEN__LENGTH__MASK;\n}\n\n#ifdef __cplusplus\n\ninline uint64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_major() const {\n return wuffs_base__token__value_major(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__to" +
|
||||
"ken__value_base_detail(this);\n}\n\ninline bool //\nwuffs_base__token::link_prev() const {\n return wuffs_base__token__link_prev(this);\n}\n\ninline bool //\nwuffs_base__token::link_next() const {\n return wuffs_base__token__link_next(this);\n}\n\ninline uint64_t //\nwuffs_base__token::length() const {\n return wuffs_base__token__length(this);\n}\n\n#endif // __cplusplus\n\n" +
|
||||
"" +
|
||||
"// --------\n\ntypedef WUFFS_BASE__SLICE(wuffs_base__token) wuffs_base__slice_token;\n\nstatic inline wuffs_base__slice_token //\nwuffs_base__make_slice_token(wuffs_base__token* ptr, size_t len) {\n wuffs_base__slice_token ret;\n ret.ptr = ptr;\n ret.len = len;\n return ret;\n}\n\n" +
|
||||
"" +
|
||||
|
||||
@@ -326,9 +326,13 @@ var Funcs = []string{
|
||||
|
||||
// ---- token_writer
|
||||
|
||||
"token_writer.write_token?(value: u64[..= 0xFFFFFFFFFFFF], length: u64[..= 0xFFFF])",
|
||||
"token_writer.write_token?(" +
|
||||
"value_major: u32[..= 0x1F_FFFF], value_minor: u32[..= 0xFF_FFFF]," +
|
||||
"link: u32[..= 0x3], length: u32[..= 0xFFFF])",
|
||||
|
||||
"token_writer.write_fast_token!(value: u64[..= 0xFFFFFFFFFFFF], length: u64[..= 0xFFFF])",
|
||||
"token_writer.write_fast_token!(" +
|
||||
"value_major: u32[..= 0x1F_FFFF], value_minor: u32[..= 0xFF_FFFF]," +
|
||||
"link: u32[..= 0x3], length: u32[..= 0xFFFF])",
|
||||
|
||||
"token_writer.available() u64",
|
||||
|
||||
|
||||
@@ -1733,22 +1733,52 @@ wuffs_base__io_buffer::writer_io_position() const {
|
||||
// ---------------- Tokens
|
||||
|
||||
typedef struct {
|
||||
// The repr is divided as:
|
||||
// - Bits 63 .. 40 (24 bits) is the major value.
|
||||
// - Bits 39 .. 16 (24 bits) is the minor value.
|
||||
// The repr's 64 bits are divided as:
|
||||
//
|
||||
// +-----+-------------+-------+-------------+-----+-----+-----------+
|
||||
// | 1 | 21 | 3 | 21 | 1 | 1 | 16 |
|
||||
// +-----+-------------+-------+-------------+-----+-----+-----------+
|
||||
// [..................value..................] LP LN length
|
||||
// [..0..|.value_major.|.....value_minor.....]
|
||||
// [..0..|.........VBC.........|.....VBD.....]
|
||||
//
|
||||
// The broad divisions are:
|
||||
// - Bits 63 .. 18 (46 bits) is the value.
|
||||
// - Bits 17 .. 16 ( 2 bits) is LP and LN (link_prev and link_next).
|
||||
// - Bits 15 .. 0 (16 bits) is the length.
|
||||
//
|
||||
// The major value is a [Base38](doc/note/base38-and-fourcc.md) value. If
|
||||
// zero (special cased for Wuffs' built-in "base" package) then the minor
|
||||
// value is further sub-divided:
|
||||
// - Bits 39 .. 37 ( 3 bits) is the value_base_category.
|
||||
// - Bits 36 .. 16 (21 bits) is the value_base_detail.
|
||||
// The value bits can be sub-divided in multiple ways:
|
||||
// - Bits 63 .. 63 ( 1 bits) is reserved (a zero bit).
|
||||
// - Bits 62 .. 42 (21 bits) is the value_major.
|
||||
// - Bits 41 .. 18 (24 bits) is the value_minor.
|
||||
// - Bits 62 .. 39 (24 bits) is the VBC (value_base_category).
|
||||
// - Bits 38 .. 18 (21 bits) is the VBD (value_base_detail).
|
||||
//
|
||||
// In particular, at 21 bits, the value_base_detail can hold every valid
|
||||
// Unicode code point.
|
||||
// The value_major is a 21-bit [Base38](doc/note/base38-and-fourcc.md) value.
|
||||
// If all of its bits are zero (special cased for Wuffs' built-in "base"
|
||||
// package) then the value_minor is further sub-divided:
|
||||
// - Bits 41 .. 39 ( 3 bits) is the VBC (value_base_category).
|
||||
// - Bits 38 .. 18 (21 bits) is the VBD (value_base_detail).
|
||||
//
|
||||
// If the major value is non-zero then the minor value has whatever arbitrary
|
||||
// meaning the tokenizer's package assigns to it.
|
||||
// The high 46 bits (bits 63 .. 18) only have VBC and VBD semantics when the
|
||||
// high 22 bits (the value_major) are all zero. An equivalent test is that
|
||||
// the high 25 bits (the notional VBC) has a numerical value less than 8.
|
||||
//
|
||||
// At 21 bits, the VBD can hold every valid Unicode code point.
|
||||
//
|
||||
// If value_major is non-zero then value_minor has whatever arbitrary meaning
|
||||
// the tokenizer's package assigns to it.
|
||||
//
|
||||
// Multiple consecutive tokens can form a larger conceptual unit. For
|
||||
// example, an "abc\tz" string is a single higher level concept but at the
|
||||
// lower level, it could consist of multiple tokens: the quotes '"', the
|
||||
// ASCII texts "abc" and "z" and the backslash-escaped tab '\t'. The LP and
|
||||
// LN (link_prev and link_next) bits denote tokens that are part of a
|
||||
// multi-token chain:
|
||||
// - LP means that this token is not the first (there is a previous token).
|
||||
// - LN means that this token is not the last (there is a next token).
|
||||
//
|
||||
// In particular, a stand-alone token will have both link bits set to zero.
|
||||
uint64_t repr;
|
||||
|
||||
#ifdef __cplusplus
|
||||
@@ -1757,6 +1787,8 @@ typedef struct {
|
||||
inline uint64_t value_minor() const;
|
||||
inline uint64_t value_base_category() const;
|
||||
inline uint64_t value_base_detail() const;
|
||||
inline bool link_prev() const;
|
||||
inline bool link_next() const;
|
||||
inline uint64_t length() const;
|
||||
#endif // __cplusplus
|
||||
|
||||
@@ -1771,20 +1803,25 @@ wuffs_base__make_token(uint64_t repr) {
|
||||
|
||||
// --------
|
||||
|
||||
#define WUFFS_BASE__TOKEN__VALUE__MASK 0xFFFFFFFFFFFF
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0xFFFFFF
|
||||
#define WUFFS_BASE__TOKEN__VALUE__MASK 0x3FFFFFFFFFFF
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0x3FFFFF
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF
|
||||
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x7FFFFFF
|
||||
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x1FFFFFF
|
||||
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF
|
||||
#define WUFFS_BASE__TOKEN__LINK__MASK 0x3
|
||||
#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF
|
||||
|
||||
#define WUFFS_BASE__TOKEN__VALUE__SHIFT 16
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 40
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 16
|
||||
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 37
|
||||
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16
|
||||
#define WUFFS_BASE__TOKEN__VALUE__SHIFT 18
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 18
|
||||
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 39
|
||||
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 18
|
||||
#define WUFFS_BASE__TOKEN__LINK__SHIFT 16
|
||||
#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0
|
||||
|
||||
#define WUFFS_BASE__TOKEN__LINK_PREV 0x20000
|
||||
#define WUFFS_BASE__TOKEN__LINK_NEXT 0x10000
|
||||
|
||||
// --------
|
||||
|
||||
#define WUFFS_BASE__TOKEN__VBC__FILLER 0
|
||||
@@ -1793,41 +1830,19 @@ wuffs_base__make_token(uint64_t repr) {
|
||||
#define WUFFS_BASE__TOKEN__VBC__NUMBER 3
|
||||
#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 4
|
||||
|
||||
// --------
|
||||
// --------
|
||||
|
||||
// INCOMPLETE means that this token combines with the following token.
|
||||
//
|
||||
// For example, tokenizing a comment that is longer than the source buffer can
|
||||
// result in multiple VBC__FILLER tokens. All but the last one is INCOMPLETE.
|
||||
//
|
||||
// By convention, whitespace is not marked incomplete. Two whitespace tokens of
|
||||
// length 30 are equivalent to one whitespace token of length 60.
|
||||
//
|
||||
// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between
|
||||
// VBD_FILLER and VBD_STRING.
|
||||
#define WUFFS_BASE__TOKEN__VBD__FILLER__INCOMPLETE 0x00001
|
||||
|
||||
#define WUFFS_BASE__TOKEN__VBD__FILLER__END_OF_CONSECUTIVE_COMMENTS 0x00010
|
||||
#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00020
|
||||
#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00040
|
||||
#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00001
|
||||
#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00002
|
||||
|
||||
// --------
|
||||
|
||||
// INCOMPLETE means that this token combines with the following token.
|
||||
//
|
||||
// For example, tokenizing a string that is longer than the source buffer can
|
||||
// result in multiple VBC__STRING tokens. All but the last one is INCOMPLETE.
|
||||
//
|
||||
// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between
|
||||
// VBD_FILLER and VBD_STRING.
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE 0x00001
|
||||
|
||||
// "DEFINITELY_FOO" means that the source bytes (and also the destination
|
||||
// bytes, assuming 1_DST_1_SRC_COPY) are in the FOO format. Definitely means
|
||||
// that the lack of the bit is conservative: it is valid for all-ASCII strings
|
||||
// to have neither DEFINITELY_ASCII or DEFINITELY_UTF_8 bits set.
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00010
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00020
|
||||
// "DEFINITELY_FOO" means that the destination bytes (and also the source
|
||||
// bytes, for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that
|
||||
// the lack of the bit is conservative: it is valid for all-ASCII strings to
|
||||
// have neither DEFINITELY_UTF_8 or DEFINITELY_ASCII bits set.
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00001
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00002
|
||||
|
||||
// "CONVERT_D_DST_S_SRC" means that multiples of S source bytes (possibly
|
||||
// padded) produces multiples of D destination bytes. For example,
|
||||
@@ -1837,21 +1852,13 @@ wuffs_base__make_token(uint64_t repr) {
|
||||
// When src is the empty string, multiple conversion algorithms are applicable
|
||||
// (so these bits are not necessarily mutually exclusive), all producing the
|
||||
// same empty dst string.
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00100
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00200
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00400
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00800
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x01000
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x02000
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x04000
|
||||
|
||||
// --------
|
||||
|
||||
// UNICODE_CODE_POINT tokens have no room in their VBD for an INCOMPLETE bit.
|
||||
// All 21 bits are used to hold the Unicode code point. Such tokens preserve
|
||||
// the INCOMPLETEness of the previous token (if a FILLER or STRING).
|
||||
|
||||
#define WUFFS_BASE__TOKEN__VBD__UNICODE_CODE_POINT__MAX_INCL 0x10FFFF
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00010
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00020
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00040
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00080
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x00100
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x00200
|
||||
#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x00400
|
||||
|
||||
// --------
|
||||
|
||||
@@ -1917,6 +1924,16 @@ wuffs_base__token__value_base_detail(const wuffs_base__token* t) {
|
||||
WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK;
|
||||
}
|
||||
|
||||
static inline bool //
|
||||
wuffs_base__token__link_prev(const wuffs_base__token* t) {
|
||||
return t->repr & WUFFS_BASE__TOKEN__LINK_PREV;
|
||||
}
|
||||
|
||||
static inline bool //
|
||||
wuffs_base__token__link_next(const wuffs_base__token* t) {
|
||||
return t->repr & WUFFS_BASE__TOKEN__LINK_NEXT;
|
||||
}
|
||||
|
||||
static inline uint64_t //
|
||||
wuffs_base__token__length(const wuffs_base__token* t) {
|
||||
return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) &
|
||||
@@ -1950,6 +1967,16 @@ wuffs_base__token::value_base_detail() const {
|
||||
return wuffs_base__token__value_base_detail(this);
|
||||
}
|
||||
|
||||
inline bool //
|
||||
wuffs_base__token::link_prev() const {
|
||||
return wuffs_base__token__link_prev(this);
|
||||
}
|
||||
|
||||
inline bool //
|
||||
wuffs_base__token::link_next() const {
|
||||
return wuffs_base__token__link_next(this);
|
||||
}
|
||||
|
||||
inline uint64_t //
|
||||
wuffs_base__token::length() const {
|
||||
return wuffs_base__token__length(this);
|
||||
@@ -19257,7 +19284,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
self->private_impl.active_coroutine = 0;
|
||||
wuffs_base__status status = wuffs_base__make_status(NULL);
|
||||
|
||||
uint64_t v_token_value = 0;
|
||||
uint32_t v_vminor = 0;
|
||||
uint32_t v_number_length = 0;
|
||||
uint32_t v_number_status = 0;
|
||||
uint32_t v_string_length = 0;
|
||||
@@ -19329,8 +19356,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) {
|
||||
if (v_whitespace_length > 0) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(((uint64_t)(v_whitespace_length))))
|
||||
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(v_whitespace_length))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_whitespace_length = 0;
|
||||
}
|
||||
@@ -19352,7 +19379,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
(iop_a_src += 1, wuffs_base__make_empty_struct());
|
||||
if (v_whitespace_length >= 65534) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(65535)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_whitespace_length = 0;
|
||||
goto label__outer__continue;
|
||||
@@ -19362,8 +19389,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
label__ws__break:;
|
||||
if (v_whitespace_length > 0) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(((uint64_t)(v_whitespace_length))))
|
||||
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(v_whitespace_length))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_whitespace_length = 0;
|
||||
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
|
||||
@@ -19376,53 +19403,28 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
}
|
||||
if (v_class == 1) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(2097171)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
(iop_a_src += 1, wuffs_base__make_empty_struct());
|
||||
label__0__continue:;
|
||||
while (true) {
|
||||
if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) {
|
||||
if (a_src && a_src->meta.closed) {
|
||||
status = wuffs_base__make_status(wuffs_json__error__bad_input);
|
||||
goto exit;
|
||||
}
|
||||
status =
|
||||
wuffs_base__make_status(wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(3);
|
||||
goto label__0__continue;
|
||||
}
|
||||
if (wuffs_base__load_u8be__no_bounds_check(iop_a_src) != 92) {
|
||||
goto label__0__break;
|
||||
}
|
||||
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
|
||||
status =
|
||||
wuffs_base__make_status(wuffs_base__suspension__short_write);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(4);
|
||||
goto label__0__continue;
|
||||
}
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(2129713)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__0__break;
|
||||
}
|
||||
label__0__break:;
|
||||
label__string_loop__continue:;
|
||||
while (true) {
|
||||
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
|
||||
status =
|
||||
wuffs_base__make_status(wuffs_base__suspension__short_write);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(5);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(3);
|
||||
goto label__string_loop__continue;
|
||||
}
|
||||
v_string_length = 0;
|
||||
label__1__continue:;
|
||||
label__0__continue:;
|
||||
while (true) {
|
||||
if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) {
|
||||
if (v_string_length > 0) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(2097697))
|
||||
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(((uint64_t)(v_string_length))))
|
||||
(((uint64_t)(2097185))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(v_string_length))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
}
|
||||
@@ -19433,7 +19435,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
}
|
||||
status =
|
||||
wuffs_base__make_status(wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(6);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(4);
|
||||
v_string_length = 0;
|
||||
goto label__string_loop__continue;
|
||||
}
|
||||
@@ -19443,28 +19445,34 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
(iop_a_src += 1, wuffs_base__make_empty_struct());
|
||||
if (v_string_length >= 65531) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(2097697))
|
||||
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(2097185))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(65532))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
goto label__string_loop__continue;
|
||||
}
|
||||
v_string_length += 1;
|
||||
goto label__1__continue;
|
||||
goto label__0__continue;
|
||||
} else if (v_char == 1) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(2097696)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(((uint64_t)(v_string_length))))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
if (v_string_length != 0) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(2097185))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(v_string_length))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
}
|
||||
goto label__string_loop__break;
|
||||
} else if (v_char == 2) {
|
||||
if (v_string_length > 0) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(2097697))
|
||||
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(((uint64_t)(v_string_length))))
|
||||
(((uint64_t)(2097185))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(v_string_length))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
|
||||
@@ -19479,7 +19487,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
}
|
||||
status = wuffs_base__make_status(
|
||||
wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(7);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(5);
|
||||
v_string_length = 0;
|
||||
v_char = 0;
|
||||
goto label__string_loop__continue;
|
||||
@@ -19491,8 +19499,9 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
(iop_a_src += 2, wuffs_base__make_empty_struct());
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(
|
||||
(4194304 | ((uint64_t)((v_backslash & 127))))))
|
||||
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(4194304 | ((uint32_t)((v_backslash & 127))))))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__string_loop__continue;
|
||||
} else if (v_c == 117) {
|
||||
@@ -19504,7 +19513,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
}
|
||||
status = wuffs_base__make_status(
|
||||
wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(8);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(6);
|
||||
v_string_length = 0;
|
||||
v_char = 0;
|
||||
goto label__string_loop__continue;
|
||||
@@ -19535,8 +19544,9 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
} else if ((v_uni4_value < 55296) || (57343 < v_uni4_value)) {
|
||||
(iop_a_src += 6, wuffs_base__make_empty_struct());
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)((4194304 | ((uint64_t)(v_uni4_value)))))
|
||||
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)((4194304 | v_uni4_value)))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(6)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__string_loop__continue;
|
||||
} else if (v_uni4_value >= 56320) {
|
||||
@@ -19549,7 +19559,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
}
|
||||
status = wuffs_base__make_status(
|
||||
wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(9);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(7);
|
||||
v_string_length = 0;
|
||||
v_uni4_value = 0;
|
||||
v_char = 0;
|
||||
@@ -19592,10 +19602,10 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
v_uni4_value -= 56320;
|
||||
(iop_a_src += 8, wuffs_base__make_empty_struct());
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(
|
||||
(4194304 | ((uint64_t)((v_uni4_high_surrogate +
|
||||
v_uni4_value))))))
|
||||
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)((4194304 | v_uni4_high_surrogate |
|
||||
v_uni4_value)))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(12))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__string_loop__continue;
|
||||
@@ -19626,7 +19636,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
}
|
||||
status = wuffs_base__make_status(
|
||||
wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(8);
|
||||
v_string_length = 0;
|
||||
v_char = 0;
|
||||
goto label__string_loop__continue;
|
||||
@@ -19639,15 +19649,16 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
(iop_a_src += 2, wuffs_base__make_empty_struct());
|
||||
if (v_string_length >= 65528) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(2097697))
|
||||
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(((uint64_t)((v_string_length + 2)))))
|
||||
(((uint64_t)(2097185))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)((v_string_length + 2)))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
goto label__string_loop__continue;
|
||||
}
|
||||
v_string_length += 2;
|
||||
goto label__1__continue;
|
||||
goto label__0__continue;
|
||||
}
|
||||
} else if (v_char == 4) {
|
||||
if (((uint64_t)(io2_a_src - iop_a_src)) < 3) {
|
||||
@@ -19658,7 +19669,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
}
|
||||
status = wuffs_base__make_status(
|
||||
wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(9);
|
||||
v_string_length = 0;
|
||||
v_char = 0;
|
||||
goto label__string_loop__continue;
|
||||
@@ -19675,15 +19686,16 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
(iop_a_src += 3, wuffs_base__make_empty_struct());
|
||||
if (v_string_length >= 65528) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(2097697))
|
||||
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(((uint64_t)((v_string_length + 3)))))
|
||||
(((uint64_t)(2097185))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)((v_string_length + 3)))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
goto label__string_loop__continue;
|
||||
}
|
||||
v_string_length += 3;
|
||||
goto label__1__continue;
|
||||
goto label__0__continue;
|
||||
}
|
||||
}
|
||||
} else if (v_char == 5) {
|
||||
@@ -19695,7 +19707,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
}
|
||||
status = wuffs_base__make_status(
|
||||
wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(12);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10);
|
||||
v_string_length = 0;
|
||||
v_char = 0;
|
||||
goto label__string_loop__continue;
|
||||
@@ -19712,22 +19724,25 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
(iop_a_src += 4, wuffs_base__make_empty_struct());
|
||||
if (v_string_length >= 65528) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(2097697))
|
||||
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(((uint64_t)((v_string_length + 4)))))
|
||||
(((uint64_t)(2097185))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)((v_string_length + 4)))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
goto label__string_loop__continue;
|
||||
}
|
||||
v_string_length += 4;
|
||||
goto label__1__continue;
|
||||
goto label__0__continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (v_string_length > 0) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(2097697)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(((uint64_t)(v_string_length))))
|
||||
(((uint64_t)(2097185))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(v_string_length))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
}
|
||||
@@ -19741,7 +19756,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
}
|
||||
}
|
||||
label__string_loop__break:;
|
||||
label__2__continue:;
|
||||
label__1__continue:;
|
||||
while (true) {
|
||||
if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) {
|
||||
if (a_src && a_src->meta.closed) {
|
||||
@@ -19750,22 +19765,24 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
}
|
||||
status =
|
||||
wuffs_base__make_status(wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(13);
|
||||
goto label__2__continue;
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11);
|
||||
goto label__1__continue;
|
||||
}
|
||||
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
|
||||
status =
|
||||
wuffs_base__make_status(wuffs_base__suspension__short_write);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(14);
|
||||
goto label__2__continue;
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(12);
|
||||
goto label__1__continue;
|
||||
}
|
||||
(iop_a_src += 1, wuffs_base__make_empty_struct());
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(2097171))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__2__break;
|
||||
goto label__1__break;
|
||||
}
|
||||
label__2__break:;
|
||||
label__1__break:;
|
||||
if (0 == (v_expect & 16)) {
|
||||
v_expect = 8;
|
||||
goto label__outer__continue;
|
||||
@@ -19774,7 +19791,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
} else if (v_class == 2) {
|
||||
(iop_a_src += 1, wuffs_base__make_empty_struct());
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
if (0 == (v_expect & 256)) {
|
||||
v_expect = 2;
|
||||
@@ -19785,7 +19802,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
} else if (v_class == 3) {
|
||||
(iop_a_src += 1, wuffs_base__make_empty_struct());
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_expect = 3762;
|
||||
goto label__outer__continue;
|
||||
@@ -19799,18 +19816,18 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
iop_a_src = a_src->data.ptr + a_src->meta.ri;
|
||||
}
|
||||
v_number_status = (v_number_length >> 8);
|
||||
v_token_value = 6291460;
|
||||
v_vminor = 6291460;
|
||||
if ((v_number_length & 128) != 0) {
|
||||
v_token_value = 6291458;
|
||||
v_vminor = 6291458;
|
||||
}
|
||||
v_number_length = (v_number_length & 127);
|
||||
if (v_number_status == 0) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(v_token_value))
|
||||
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(((uint64_t)(v_number_length))))
|
||||
(((uint64_t)(v_vminor))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(v_number_length))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__3__break;
|
||||
goto label__2__break;
|
||||
}
|
||||
while (v_number_length > 0) {
|
||||
v_number_length -= 1;
|
||||
@@ -19832,23 +19849,23 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
} else {
|
||||
status =
|
||||
wuffs_base__make_status(wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(15);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(13);
|
||||
while (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
|
||||
status = wuffs_base__make_status(
|
||||
wuffs_base__suspension__short_write);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(16);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(14);
|
||||
}
|
||||
}
|
||||
}
|
||||
label__3__break:;
|
||||
label__2__break:;
|
||||
goto label__goto_parsed_a_leaf_value__break;
|
||||
} else if (v_class == 5) {
|
||||
v_token_value = 8405009;
|
||||
v_vminor = 8405009;
|
||||
if (v_depth == 0) {
|
||||
} else if (0 != (v_expect_after_value & 64)) {
|
||||
v_token_value = 8405057;
|
||||
v_vminor = 8405057;
|
||||
} else {
|
||||
v_token_value = 8405025;
|
||||
v_vminor = 8405025;
|
||||
}
|
||||
if (v_depth >= 1024) {
|
||||
status = wuffs_base__make_status(
|
||||
@@ -19862,7 +19879,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
v_depth += 1;
|
||||
(iop_a_src += 1, wuffs_base__make_empty_struct());
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(v_token_value)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(v_vminor))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_expect = 66;
|
||||
v_expect_after_value = 68;
|
||||
@@ -19871,7 +19889,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
(iop_a_src += 1, wuffs_base__make_empty_struct());
|
||||
if (v_depth <= 1) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(8392770)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(8392770))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__outer__break;
|
||||
}
|
||||
@@ -19881,25 +19900,27 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
if (0 == (self->private_data.f_stack[v_stack_byte] &
|
||||
(((uint32_t)(1)) << v_stack_bit))) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(8396866)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(8396866))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_expect = 260;
|
||||
v_expect_after_value = 260;
|
||||
} else {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(8405058)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(8405058))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_expect = 68;
|
||||
v_expect_after_value = 68;
|
||||
}
|
||||
goto label__outer__continue;
|
||||
} else if (v_class == 7) {
|
||||
v_token_value = 8396817;
|
||||
v_vminor = 8396817;
|
||||
if (v_depth == 0) {
|
||||
} else if (0 != (v_expect_after_value & 64)) {
|
||||
v_token_value = 8396865;
|
||||
v_vminor = 8396865;
|
||||
} else {
|
||||
v_token_value = 8396833;
|
||||
v_vminor = 8396833;
|
||||
}
|
||||
if (v_depth >= 1024) {
|
||||
status = wuffs_base__make_status(
|
||||
@@ -19913,7 +19934,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
v_depth += 1;
|
||||
(iop_a_src += 1, wuffs_base__make_empty_struct());
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(v_token_value)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(v_vminor))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_expect = 4018;
|
||||
v_expect_after_value = 260;
|
||||
@@ -19922,7 +19944,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
(iop_a_src += 1, wuffs_base__make_empty_struct());
|
||||
if (v_depth <= 1) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(8392738)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(8392738))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__outer__break;
|
||||
}
|
||||
@@ -19932,13 +19955,15 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
if (0 == (self->private_data.f_stack[v_stack_byte] &
|
||||
(((uint32_t)(1)) << v_stack_bit))) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(8396834)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(8396834))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_expect = 260;
|
||||
v_expect_after_value = 260;
|
||||
} else {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(8405026)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(8405026))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_expect = 68;
|
||||
v_expect_after_value = 68;
|
||||
@@ -19949,7 +19974,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
111546413966853);
|
||||
if (v_match == 0) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(6292481)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(6292481))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(5)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
if (((uint64_t)(io2_a_src - iop_a_src)) < 5) {
|
||||
status = wuffs_base__make_status(
|
||||
@@ -19961,7 +19987,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
} else if (v_match == 1) {
|
||||
status =
|
||||
wuffs_base__make_status(wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(17);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(15);
|
||||
goto label__outer__continue;
|
||||
}
|
||||
} else if (v_class == 10) {
|
||||
@@ -19969,7 +19995,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
435762131972);
|
||||
if (v_match == 0) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(6293505)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(6293505))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(4)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
if (((uint64_t)(io2_a_src - iop_a_src)) < 4) {
|
||||
status = wuffs_base__make_status(
|
||||
@@ -19981,7 +20008,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
} else if (v_match == 1) {
|
||||
status =
|
||||
wuffs_base__make_status(wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(18);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(16);
|
||||
goto label__outer__continue;
|
||||
}
|
||||
} else if (v_class == 11) {
|
||||
@@ -19989,7 +20016,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
465676103172);
|
||||
if (v_match == 0) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(6291969)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
|
||||
(((uint64_t)(6291969))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(4)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
if (((uint64_t)(io2_a_src - iop_a_src)) < 4) {
|
||||
status = wuffs_base__make_status(
|
||||
@@ -20001,7 +20029,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
} else if (v_match == 1) {
|
||||
status =
|
||||
wuffs_base__make_status(wuffs_base__suspension__short_read);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(19);
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(17);
|
||||
goto label__outer__continue;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,23 +17,38 @@
|
||||
// print-json-token-debug-format.c parses JSON from stdin and prints the
|
||||
// resulting token stream, eliding any non-essential (e.g. whitespace) tokens.
|
||||
//
|
||||
// The output format is only for debugging, and certainly not for long term
|
||||
// storage. It isn't guaranteed to be stable between versions of this program
|
||||
// and of the Wuffs standard library.
|
||||
// The output format is only for debugging or regression testing, and certainly
|
||||
// not for long term storage. It isn't guaranteed to be stable between versions
|
||||
// of this program and of the Wuffs standard library.
|
||||
//
|
||||
// It prints 16 bytes (128 bits) per token, containing big-endian numbers:
|
||||
//
|
||||
// +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
|
||||
// | POS | LEN | LP| LN| MAJOR | MINOR |
|
||||
// | | | | | |VBC| VBD |
|
||||
// +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
|
||||
//
|
||||
// - POS (4 bytes) is the position: the sum of all previous tokens' lengths,
|
||||
// including elided tokens.
|
||||
// - LEN (2 bytes) is the length.
|
||||
// - LP (1 bytes) is the link_prev bit.
|
||||
// - LN (1 bytes) is the link_next bit
|
||||
// - MAJOR (4 bytes) is the value_major.
|
||||
//
|
||||
// The final 4 bytes are either value_minor (when the value_major is non-zero)
|
||||
// or 1 + 3 bytes for value_base_category and value_base_detail (otherwise).
|
||||
//
|
||||
// It prints 16 bytes (4 big-endian uint32's) per token: the token's position
|
||||
// (total length of all previous tokens), length, major value and minor value.
|
||||
// Together with the hexadecimal WUFFS_BASE__TOKEN__ETC constants defined in
|
||||
// token-public.h, this format is somewhat human-readable when piped through a
|
||||
// hex-dump program (e.g. /usr/bin/hd).
|
||||
// hex-dump program (such as /usr/bin/hd), printing one token per line.
|
||||
//
|
||||
// If the input or output is larger than the program's buffers (64 MiB and
|
||||
// 131072 tokens by default), there may be multiple valid tokenizations of any
|
||||
// given input. For example, if a source string "abcde" straddles an I/O
|
||||
// boundary, it may be tokenized as a 5-length complete string or as a 3-length
|
||||
// incomplete string followed by a 2-length complete string.
|
||||
// boundary, it may be tokenized as single (no-link) 5-length string or as a
|
||||
// 3-length link_next'ed string followed by a 2-length link_prev'ed string.
|
||||
//
|
||||
// A Wuffs token stream, in general, can support inputs more than (1 << 32)
|
||||
// A Wuffs token stream, in general, can support inputs more than 0xFFFF_FFFF
|
||||
// bytes long, but this program can not, as it tracks the tokens' cumulative
|
||||
// position as a uint32.
|
||||
|
||||
@@ -134,17 +149,28 @@ main1(int argc, char** argv) {
|
||||
|
||||
while (tok.meta.ri < tok.meta.wi) {
|
||||
wuffs_base__token* t = &tok.data.ptr[tok.meta.ri++];
|
||||
uint64_t len = wuffs_base__token__length(t);
|
||||
uint16_t len = wuffs_base__token__length(t);
|
||||
|
||||
if (wuffs_base__token__value(t) != 0) {
|
||||
uint64_t maj = wuffs_base__token__value_major(t);
|
||||
uint64_t min = wuffs_base__token__value_minor(t);
|
||||
uint8_t lp = wuffs_base__token__link_prev(t) ? 1 : 0;
|
||||
uint8_t ln = wuffs_base__token__link_next(t) ? 1 : 0;
|
||||
uint32_t vmajor = wuffs_base__token__value_major(t);
|
||||
|
||||
uint8_t buf[16];
|
||||
wuffs_base__store_u32be__no_bounds_check(&buf[0 * 4], (uint32_t)(pos));
|
||||
wuffs_base__store_u32be__no_bounds_check(&buf[1 * 4], (uint32_t)(len));
|
||||
wuffs_base__store_u32be__no_bounds_check(&buf[2 * 4], (uint32_t)(maj));
|
||||
wuffs_base__store_u32be__no_bounds_check(&buf[3 * 4], (uint32_t)(min));
|
||||
wuffs_base__store_u32be__no_bounds_check(&buf[0x0], (uint32_t)(pos));
|
||||
wuffs_base__store_u16be__no_bounds_check(&buf[0x4], len);
|
||||
wuffs_base__store_u8__no_bounds_check(&buf[0x0006], lp);
|
||||
wuffs_base__store_u8__no_bounds_check(&buf[0x0007], ln);
|
||||
wuffs_base__store_u32be__no_bounds_check(&buf[0x8], vmajor);
|
||||
if (vmajor) {
|
||||
uint32_t vminor = wuffs_base__token__value_minor(t);
|
||||
wuffs_base__store_u32be__no_bounds_check(&buf[0xC], vminor);
|
||||
} else {
|
||||
uint8_t vbc = wuffs_base__token__value_base_category(t);
|
||||
uint32_t vbd = wuffs_base__token__value_base_detail(t);
|
||||
wuffs_base__store_u8__no_bounds_check(&buf[0x000C], vbc);
|
||||
wuffs_base__store_u24be__no_bounds_check(&buf[0xD], vbd);
|
||||
}
|
||||
const int stdout_fd = 1;
|
||||
write(stdout_fd, &buf[0], 16);
|
||||
}
|
||||
|
||||
+115
-78
@@ -45,7 +45,7 @@ pub struct decoder? implements base.token_decoder(
|
||||
)
|
||||
|
||||
pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
var token_value : base.u64[..= 0xFF_FFFF]
|
||||
var vminor : base.u32[..= 0xFF_FFFF]
|
||||
var number_length : base.u32[..= 0x3FF]
|
||||
var number_status : base.u32[..= 0x3]
|
||||
var string_length : base.u32[..= 0xFFFB]
|
||||
@@ -95,8 +95,10 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
if args.src.available() <= 0 {
|
||||
if whitespace_length > 0 {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0,
|
||||
length: whitespace_length as base.u64)
|
||||
value_major: 0,
|
||||
value_minor: 0,
|
||||
link: 0x0,
|
||||
length: whitespace_length)
|
||||
whitespace_length = 0
|
||||
}
|
||||
if args.src.is_closed() {
|
||||
@@ -116,7 +118,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
|
||||
if whitespace_length >= 0xFFFE {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0,
|
||||
value_major: 0,
|
||||
value_minor: 0,
|
||||
link: 0x0,
|
||||
length: 0xFFFF)
|
||||
whitespace_length = 0
|
||||
continue.outer
|
||||
@@ -127,8 +131,10 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
// Emit whitespace.
|
||||
if whitespace_length > 0 {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0,
|
||||
length: whitespace_length as base.u64)
|
||||
value_major: 0,
|
||||
value_minor: 0,
|
||||
link: 0x0,
|
||||
length: whitespace_length)
|
||||
whitespace_length = 0
|
||||
if args.dst.available() <= 0 {
|
||||
continue.outer
|
||||
@@ -149,37 +155,14 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
|
||||
if class == 0x01 { // 0x01 is CLASS_STRING.
|
||||
// -------- BEGIN parse strings.
|
||||
// The leading '"' is filler.
|
||||
// Emit the leading '"'.
|
||||
args.dst.write_fast_token!(
|
||||
value: 0,
|
||||
value_major: 0,
|
||||
value_minor: 0x20_0013,
|
||||
link: 0x1,
|
||||
length: 1)
|
||||
args.src.skip32_fast!(actual: 1, worst_case: 1)
|
||||
|
||||
// If the string contents starts with an escape code, issue an
|
||||
// incomplete zero-length string token (with
|
||||
// WUFFS_BASE__TOKEN__VBC__STRING) before issuing a token with
|
||||
// WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT.
|
||||
while true {
|
||||
if args.src.available() <= 0 {
|
||||
if args.src.is_closed() {
|
||||
return "#bad input"
|
||||
}
|
||||
yield? base."$short read"
|
||||
continue
|
||||
}
|
||||
if args.src.peek_u8() <> 0x5C { // 0x5C is '\\'.
|
||||
break
|
||||
}
|
||||
if args.dst.available() <= 0 {
|
||||
yield? base."$short write"
|
||||
continue
|
||||
}
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x20_7F31,
|
||||
length: 0)
|
||||
break
|
||||
}
|
||||
|
||||
while.string_loop true {
|
||||
if args.dst.available() <= 0 {
|
||||
yield? base."$short write"
|
||||
@@ -193,8 +176,10 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
if args.src.available() <= 0 {
|
||||
if string_length > 0 {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x20_0221,
|
||||
length: string_length as base.u64)
|
||||
value_major: 0,
|
||||
value_minor: 0x20_0021,
|
||||
link: 0x3,
|
||||
length: string_length)
|
||||
string_length = 0
|
||||
}
|
||||
if args.src.is_closed() {
|
||||
@@ -211,7 +196,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
args.src.skip32_fast!(actual: 1, worst_case: 1)
|
||||
if string_length >= 0xFFFB {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x20_0221,
|
||||
value_major: 0,
|
||||
value_minor: 0x20_0021,
|
||||
link: 0x3,
|
||||
length: 0xFFFC)
|
||||
string_length = 0
|
||||
continue.string_loop
|
||||
@@ -220,17 +207,23 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
continue
|
||||
|
||||
} else if char == 0x01 { // '"'
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x20_0220,
|
||||
length: string_length as base.u64)
|
||||
string_length = 0
|
||||
if string_length <> 0 {
|
||||
args.dst.write_fast_token!(
|
||||
value_major: 0,
|
||||
value_minor: 0x20_0021,
|
||||
link: 0x3,
|
||||
length: string_length)
|
||||
string_length = 0
|
||||
}
|
||||
break.string_loop
|
||||
|
||||
} else if char == 0x02 { // '\\'.
|
||||
if string_length > 0 {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x20_0221,
|
||||
length: string_length as base.u64)
|
||||
value_major: 0,
|
||||
value_minor: 0x20_0021,
|
||||
link: 0x3,
|
||||
length: string_length)
|
||||
string_length = 0
|
||||
if args.dst.available() <= 0 {
|
||||
continue.string_loop
|
||||
@@ -252,7 +245,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
if backslash > 0 {
|
||||
args.src.skip32_fast!(actual: 2, worst_case: 2)
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x40_0000 | ((backslash & 0x7F) as base.u64),
|
||||
value_major: 0,
|
||||
value_minor: 0x40_0000 | ((backslash & 0x7F) as base.u32),
|
||||
link: 0x3,
|
||||
length: 2)
|
||||
continue.string_loop
|
||||
|
||||
@@ -294,7 +289,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
// Not a Unicode surrogate. We're good.
|
||||
args.src.skip32_fast!(actual: 6, worst_case: 6)
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x40_0000 | (uni4_value as base.u64),
|
||||
value_major: 0,
|
||||
value_minor: 0x40_0000 | uni4_value,
|
||||
link: 0x3,
|
||||
length: 6)
|
||||
continue.string_loop
|
||||
|
||||
@@ -358,7 +355,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
uni4_value -= 0xDC00
|
||||
args.src.skip32_fast!(actual: 8, worst_case: 8)
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x40_0000 | ((uni4_high_surrogate + uni4_value) as base.u64),
|
||||
value_major: 0,
|
||||
value_minor: 0x40_0000 | uni4_high_surrogate | uni4_value,
|
||||
link: 0x3,
|
||||
length: 12)
|
||||
continue.string_loop
|
||||
|
||||
@@ -399,8 +398,10 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
args.src.skip32_fast!(actual: 2, worst_case: 2)
|
||||
if string_length >= 0xFFF8 {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x20_0221,
|
||||
length: (string_length + 2) as base.u64)
|
||||
value_major: 0,
|
||||
value_minor: 0x20_0021,
|
||||
link: 0x3,
|
||||
length: string_length + 2)
|
||||
string_length = 0
|
||||
continue.string_loop
|
||||
}
|
||||
@@ -429,8 +430,10 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
args.src.skip32_fast!(actual: 3, worst_case: 3)
|
||||
if string_length >= 0xFFF8 {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x20_0221,
|
||||
length: (string_length + 3) as base.u64)
|
||||
value_major: 0,
|
||||
value_minor: 0x20_0021,
|
||||
link: 0x3,
|
||||
length: string_length + 3)
|
||||
string_length = 0
|
||||
continue.string_loop
|
||||
}
|
||||
@@ -459,8 +462,10 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
args.src.skip32_fast!(actual: 4, worst_case: 4)
|
||||
if string_length >= 0xFFF8 {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x20_0221,
|
||||
length: (string_length + 4) as base.u64)
|
||||
value_major: 0,
|
||||
value_minor: 0x20_0021,
|
||||
link: 0x3,
|
||||
length: string_length + 4)
|
||||
string_length = 0
|
||||
continue.string_loop
|
||||
}
|
||||
@@ -472,8 +477,10 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
|
||||
if string_length > 0 {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x20_0221,
|
||||
length: string_length as base.u64)
|
||||
value_major: 0,
|
||||
value_minor: 0x20_0021,
|
||||
link: 0x3,
|
||||
length: string_length)
|
||||
string_length = 0
|
||||
}
|
||||
if char == 0x80 {
|
||||
@@ -483,7 +490,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
}
|
||||
} endwhile.string_loop
|
||||
|
||||
// The trailing '"' is filler.
|
||||
// Emit the trailing '"'.
|
||||
while true {
|
||||
if args.src.available() <= 0 {
|
||||
if args.src.is_closed() {
|
||||
@@ -498,7 +505,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
}
|
||||
args.src.skip32_fast!(actual: 1, worst_case: 1)
|
||||
args.dst.write_fast_token!(
|
||||
value: 0,
|
||||
value_major: 0,
|
||||
value_minor: 0x20_0013,
|
||||
link: 0x2,
|
||||
length: 1)
|
||||
break
|
||||
}
|
||||
@@ -517,7 +526,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
args.src.skip32_fast!(actual: 1, worst_case: 1)
|
||||
// The ',' is filler.
|
||||
args.dst.write_fast_token!(
|
||||
value: 0,
|
||||
value_major: 0,
|
||||
value_minor: 0,
|
||||
link: 0x0,
|
||||
length: 1)
|
||||
// What's valid after a comma depends on whether or not we're
|
||||
// in an array or an object.
|
||||
@@ -532,7 +543,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
args.src.skip32_fast!(actual: 1, worst_case: 1)
|
||||
// The ':' is filler.
|
||||
args.dst.write_fast_token!(
|
||||
value: 0,
|
||||
value_major: 0,
|
||||
value_minor: 0,
|
||||
link: 0x0,
|
||||
length: 1)
|
||||
expect = 0x0EB2 // 0x0EB2 is EXPECT_VALUE.
|
||||
continue.outer
|
||||
@@ -544,15 +557,17 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
{
|
||||
number_length = this.decode_number!(src: args.src)
|
||||
number_status = number_length >> 8
|
||||
token_value = 0x60_0004
|
||||
vminor = 0x60_0004
|
||||
if (number_length & 0x80) <> 0 {
|
||||
token_value = 0x60_0002
|
||||
vminor = 0x60_0002
|
||||
}
|
||||
number_length = number_length & 0x7F
|
||||
if number_status == 0 {
|
||||
args.dst.write_fast_token!(
|
||||
value: token_value,
|
||||
length: number_length as base.u64)
|
||||
value_major: 0,
|
||||
value_minor: vminor,
|
||||
link: 0x0,
|
||||
length: number_length)
|
||||
break
|
||||
}
|
||||
|
||||
@@ -582,13 +597,13 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
// -------- END parse numbers.
|
||||
|
||||
} else if class == 0x05 { // 0x05 is CLASS_OPEN_CURLY_BRACE.
|
||||
token_value = 0x80_4011
|
||||
vminor = 0x80_4011
|
||||
if depth == 0 {
|
||||
// No-op.
|
||||
} else if 0 <> (expect_after_value & 0x0040) { // 0x0040 is EXPECT_CLOSE_CURLY_BRACE.
|
||||
token_value = 0x80_4041
|
||||
vminor = 0x80_4041
|
||||
} else {
|
||||
token_value = 0x80_4021
|
||||
vminor = 0x80_4021
|
||||
}
|
||||
if depth >= 1024 {
|
||||
return "#unsupported recursion depth"
|
||||
@@ -600,7 +615,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
|
||||
args.src.skip32_fast!(actual: 1, worst_case: 1)
|
||||
args.dst.write_fast_token!(
|
||||
value: token_value,
|
||||
value_major: 0,
|
||||
value_minor: vminor,
|
||||
link: 0x0,
|
||||
length: 1)
|
||||
expect = 0x0042 // 0x0042 is (EXPECT_CLOSE_CURLY_BRACE | EXPECT_STRING).
|
||||
expect_after_value = 0x0044 // 0x0044 is (EXPECT_CURLY_CLOSE_BRACE | EXPECT_COMMA).
|
||||
@@ -610,7 +627,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
args.src.skip32_fast!(actual: 1, worst_case: 1)
|
||||
if depth <= 1 {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x80_1042,
|
||||
value_major: 0,
|
||||
value_minor: 0x80_1042,
|
||||
link: 0x0,
|
||||
length: 1)
|
||||
break.outer
|
||||
}
|
||||
@@ -619,14 +638,18 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
stack_bit = (depth - 1) & 31
|
||||
if 0 == (this.stack[stack_byte] & ((1 as base.u32) << stack_bit)) {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x80_2042,
|
||||
value_major: 0,
|
||||
value_minor: 0x80_2042,
|
||||
link: 0x0,
|
||||
length: 1)
|
||||
// 0x0104 is (EXPECT_SQUARE_CLOSE_BRACKET | EXPECT_COMMA).
|
||||
expect = 0x0104
|
||||
expect_after_value = 0x0104
|
||||
} else {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x80_4042,
|
||||
value_major: 0,
|
||||
value_minor: 0x80_4042,
|
||||
link: 0x0,
|
||||
length: 1)
|
||||
// 0x0044 is (EXPECT_CURLY_CLOSE_BRACE | EXPECT_COMMA).
|
||||
expect = 0x0044
|
||||
@@ -635,13 +658,13 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
continue.outer
|
||||
|
||||
} else if class == 0x07 { // 0x07 is CLASS_OPEN_SQUARE_BRACKET.
|
||||
token_value = 0x80_2011
|
||||
vminor = 0x80_2011
|
||||
if depth == 0 {
|
||||
// No-op.
|
||||
} else if 0 <> (expect_after_value & 0x0040) { // 0x0040 is EXPECT_CLOSE_CURLY_BRACE.
|
||||
token_value = 0x80_2041
|
||||
vminor = 0x80_2041
|
||||
} else {
|
||||
token_value = 0x80_2021
|
||||
vminor = 0x80_2021
|
||||
}
|
||||
if depth >= 1024 {
|
||||
return "#unsupported recursion depth"
|
||||
@@ -653,7 +676,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
|
||||
args.src.skip32_fast!(actual: 1, worst_case: 1)
|
||||
args.dst.write_fast_token!(
|
||||
value: token_value,
|
||||
value_major: 0,
|
||||
value_minor: vminor,
|
||||
link: 0x0,
|
||||
length: 1)
|
||||
expect = 0x0FB2 // 0x0FB2 is (EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_VALUE).
|
||||
expect_after_value = 0x0104 // 0x0104 is (EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA).
|
||||
@@ -663,7 +688,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
args.src.skip32_fast!(actual: 1, worst_case: 1)
|
||||
if depth <= 1 {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x80_1022,
|
||||
value_major: 0,
|
||||
value_minor: 0x80_1022,
|
||||
link: 0x0,
|
||||
length: 1)
|
||||
break.outer
|
||||
}
|
||||
@@ -672,14 +699,18 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
stack_bit = (depth - 1) & 31
|
||||
if 0 == (this.stack[stack_byte] & ((1 as base.u32) << stack_bit)) {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x80_2022,
|
||||
value_major: 0,
|
||||
value_minor: 0x80_2022,
|
||||
link: 0x0,
|
||||
length: 1)
|
||||
// 0x0104 is (EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA).
|
||||
expect = 0x0104
|
||||
expect_after_value = 0x0104
|
||||
} else {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x80_4022,
|
||||
value_major: 0,
|
||||
value_minor: 0x80_4022,
|
||||
link: 0x0,
|
||||
length: 1)
|
||||
// 0x0044 is (EXPECT_CLOSE_CURLY_BRACE | EXPECT_COMMA).
|
||||
expect = 0x0044
|
||||
@@ -691,7 +722,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
match = args.src.match7(a: 0x6573_6C61_6605) // 5 bytes "false".
|
||||
if match == 0 {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x60_0401,
|
||||
value_major: 0,
|
||||
value_minor: 0x60_0401,
|
||||
link: 0x0,
|
||||
length: 5)
|
||||
if args.src.available() < 5 {
|
||||
return "#internal error: inconsistent I/O"
|
||||
@@ -707,7 +740,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
match = args.src.match7(a: 0x65_7572_7404) // 4 bytes "true".
|
||||
if match == 0 {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x60_0801,
|
||||
value_major: 0,
|
||||
value_minor: 0x60_0801,
|
||||
link: 0x0,
|
||||
length: 4)
|
||||
if args.src.available() < 4 {
|
||||
return "#internal error: inconsistent I/O"
|
||||
@@ -723,7 +758,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
|
||||
match = args.src.match7(a: 0x6C_6C75_6E04) // 4 bytes "null".
|
||||
if match == 0 {
|
||||
args.dst.write_fast_token!(
|
||||
value: 0x60_0201,
|
||||
value_major: 0,
|
||||
value_minor: 0x60_0201,
|
||||
link: 0x0,
|
||||
length: 4)
|
||||
if args.src.available() < 4 {
|
||||
return "#internal error: inconsistent I/O"
|
||||
|
||||
@@ -1089,11 +1089,12 @@ do_test__wuffs_base__token_decoder(wuffs_base__token_decoder* b,
|
||||
uint64_t pos = 0;
|
||||
while (tok.meta.ri < tok.meta.wi) {
|
||||
wuffs_base__token* t = &tok.data.ptr[tok.meta.ri++];
|
||||
uint64_t len = wuffs_base__token__length(t);
|
||||
uint16_t len = wuffs_base__token__length(t);
|
||||
|
||||
if (wuffs_base__token__value(t) != 0) {
|
||||
uint64_t maj = wuffs_base__token__value_major(t);
|
||||
uint64_t min = wuffs_base__token__value_minor(t);
|
||||
uint8_t lp = wuffs_base__token__link_prev(t) ? 1 : 0;
|
||||
uint8_t ln = wuffs_base__token__link_next(t) ? 1 : 0;
|
||||
uint32_t vmajor = wuffs_base__token__value_major(t);
|
||||
|
||||
if ((have.data.len - have.meta.wi) < 16) {
|
||||
return "testlib: output is too long";
|
||||
@@ -1101,10 +1102,21 @@ do_test__wuffs_base__token_decoder(wuffs_base__token_decoder* b,
|
||||
// This 16-bytes-per-token debug format is the same one used by
|
||||
// `script/print-json-token-debug-format.c`.
|
||||
uint8_t* ptr = have.data.ptr + have.meta.wi;
|
||||
wuffs_base__store_u32be__no_bounds_check(ptr + (0 * 4), (uint32_t)(pos));
|
||||
wuffs_base__store_u32be__no_bounds_check(ptr + (1 * 4), (uint32_t)(len));
|
||||
wuffs_base__store_u32be__no_bounds_check(ptr + (2 * 4), (uint32_t)(maj));
|
||||
wuffs_base__store_u32be__no_bounds_check(ptr + (3 * 4), (uint32_t)(min));
|
||||
|
||||
wuffs_base__store_u32be__no_bounds_check(ptr + 0x0, (uint32_t)(pos));
|
||||
wuffs_base__store_u16be__no_bounds_check(ptr + 0x4, len);
|
||||
wuffs_base__store_u8__no_bounds_check(ptr + 0x0006, lp);
|
||||
wuffs_base__store_u8__no_bounds_check(ptr + 0x0007, ln);
|
||||
wuffs_base__store_u32be__no_bounds_check(ptr + 0x8, vmajor);
|
||||
if (vmajor) {
|
||||
uint32_t vminor = wuffs_base__token__value_minor(t);
|
||||
wuffs_base__store_u32be__no_bounds_check(ptr + 0xC, vminor);
|
||||
} else {
|
||||
uint8_t vbc = wuffs_base__token__value_base_category(t);
|
||||
uint32_t vbd = wuffs_base__token__value_base_detail(t);
|
||||
wuffs_base__store_u8__no_bounds_check(ptr + 0x000C, vbc);
|
||||
wuffs_base__store_u24be__no_bounds_check(ptr + 0xD, vbd);
|
||||
}
|
||||
have.meta.wi += 16;
|
||||
}
|
||||
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Reference in new issue
Block a user