mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-18 00:32:11 +01:00
Remove wuffs_base__token__link_prev
This commit is contained in:
1 parent
2ef3999d6d
commit
496e88b6c5
15 files changed
+206
-240
No files matched your search
+31
-32
@@ -36,25 +36,22 @@ string are treated differently when reconstructing the decoded string:
|
||||
|
||||
A token is just a `uint64_t`. The broad divisions are:
|
||||
|
||||
- Bits 63 .. 18 (46 bits) is the value.
|
||||
- Bits 17 .. 16 (2 bits) is `LP` and `LN` (`link_prev` and `link_next`).
|
||||
- Bits 63 .. 17 (47 bits) is the value.
|
||||
- Bits 16 .. 16 (1 bit) is the `continued` bit.
|
||||
- Bits 15 .. 0 (16 bits) is the length.
|
||||
|
||||
The `LP` and `LN` bits denote tokens that are part of a multi-token chain:
|
||||
|
||||
- `LP` means that this token is not the first (there is a previous token).
|
||||
- `LN` means that this token is not the last (there is a next token).
|
||||
|
||||
A stand-alone token will have both link bits set to zero.
|
||||
The `continued` bit is whether that the token chain for this token also
|
||||
contains the next token. The final token in a token chain, including
|
||||
stand-alone tokens, will have the `continued` bit set to zero.
|
||||
|
||||
```
|
||||
+-----+-------------+-------+-------------+-----+-----+-----------+
|
||||
| 1 | 21 | 3 | 21 | 1 | 1 | 16 |
|
||||
+-----+-------------+-------+-------------+-----+-----+-----------+
|
||||
[..................value..................] LP LN length
|
||||
[..1..|..........~value_extension.........]
|
||||
[..0..|.value_major.|.....value_minor.....]
|
||||
[..0..|......0......|..VBC..|.....VBD.....]
|
||||
+-----+-------------+-------+---------------+-----+-----------+
|
||||
| 1 | 21 | 3 | 22 | 1 | 16 |
|
||||
+-----+-------------+-------+---------------+-----+-----------+
|
||||
[...................value...................] con length
|
||||
[..1..|...........~value_extension..........]
|
||||
[..0..|.value_major.|......value_minor......]
|
||||
[..0..|......0......|..VBC..|......VBD......]
|
||||
```
|
||||
|
||||
The value bits can be sub-divided in multiple ways. First, the high bit:
|
||||
@@ -64,7 +61,7 @@ The value bits can be sub-divided in multiple ways. First, the high bit:
|
||||
|
||||
### Extended Tokens
|
||||
|
||||
- Bits 62 .. 18 (45 bits) is the bitwise-not (~) of the `value_extension`.
|
||||
- Bits 62 .. 17 (46 bits) is the bitwise-not (~) of the `value_extension`.
|
||||
|
||||
Extended tokens are typically part of a multi-token chain whose first token is
|
||||
a simple token that provides the semantics for each `value_extension`.
|
||||
@@ -73,7 +70,7 @@ a simple token that provides the semantics for each `value_extension`.
|
||||
### Simple Tokens
|
||||
|
||||
- Bits 62 .. 42 (21 bits) is the `value_major`.
|
||||
- Bits 41 .. 18 (24 bits) is the `value_minor`.
|
||||
- Bits 41 .. 17 (25 bits) is the `value_minor`.
|
||||
|
||||
The `value_major` is a 21-bit [Base38](doc/note/base38-and-fourcc.md) number.
|
||||
For example, an HTML tokenizer might produce a combination of "base" tokens
|
||||
@@ -90,13 +87,13 @@ tokenizer's package assigns to it.
|
||||
A zero `value_major` is reserved for Wuffs' built-in "base" package. The
|
||||
`value_minor` is further sub-divided:
|
||||
|
||||
- Bits 41 .. 39 (3 bits) is the `VBC` (`value_base_category`).
|
||||
- Bits 38 .. 18 (21 bits) is the `VBD` (`value_base_detail`).
|
||||
- Bits 41 .. 38 (4 bits) is the `VBC` (`value_base_category`).
|
||||
- Bits 37 .. 17 (21 bits) is the `VBD` (`value_base_detail`).
|
||||
|
||||
The high 46 bits (bits 63 .. 18) only have `VBC` and `VBD` semantics when the
|
||||
The high 47 bits (bits 63 .. 17) only have `VBC` and `VBD` semantics when the
|
||||
high 22 bits (the `extended` and `value_major` parts) are all zero. An
|
||||
equivalent test is that the high 25 bits (the notional `VBC`), as either an
|
||||
unsigned integer or a sign-extended integer, is in the range `0 ..= 7`.
|
||||
equivalent test is that the high 26 bits (the notional `VBC`), as either an
|
||||
unsigned integer or a sign-extended integer, is in the range `0 ..= 15`.
|
||||
|
||||
These `VBC`s organize tokens into broad groups, generally applicable (as
|
||||
opposed to being e.g. HTML-specific or JSON-specific). For example, strings and
|
||||
@@ -142,15 +139,17 @@ entire program (not just the Wuffs library) never calls `malloc`.
|
||||
## Example Token Stream
|
||||
|
||||
```
|
||||
$ gcc script/print-json-token-debug-format.c && \
|
||||
> ./a.out -all-tokens -human-readable < test/data/json-things.formatted.json
|
||||
pos=0x00000000 len=0x0001 link=0b00 vbc=1:Structure........ vbd=0x004011
|
||||
pos=0x00000001 len=0x0005 link=0b00 vbc=0:Filler........... vbd=0x000000
|
||||
pos=0x00000006 len=0x0001 link=0b01 vbc=2:String........... vbd=0x000013
|
||||
pos=0x00000007 len=0x0002 link=0b11 vbc=2:String........... vbd=0x000021
|
||||
pos=0x00000009 len=0x0001 link=0b10 vbc=2:String........... vbd=0x000013
|
||||
$ gcc script/print-json-token-debug-format.c -o pjtdf
|
||||
$ ./pjtdf -all-tokens -human-readable < test/data/json-things.formatted.json
|
||||
pos=0x00000000 len=0x0001 con=0 vbc=1:Structure........ vbd=0x004011
|
||||
pos=0x00000001 len=0x0005 con=0 vbc=0:Filler........... vbd=0x000000
|
||||
pos=0x00000006 len=0x0001 con=1 vbc=2:String........... vbd=0x000013
|
||||
pos=0x00000007 len=0x0002 con=1 vbc=2:String........... vbd=0x000021
|
||||
pos=0x00000009 len=0x0001 con=0 vbc=2:String........... vbd=0x000013
|
||||
etc
|
||||
pos=0x00000094 len=0x0001 link=0b10 vbc=2:String........... vbd=0x000013
|
||||
pos=0x00000095 len=0x0001 link=0b00 vbc=0:Filler........... vbd=0x000000
|
||||
pos=0x00000096 len=0x0001 link=0b00 vbc=1:Structure........ vbd=0x001042
|
||||
pos=0x00000090 len=0x0002 con=1 vbc=3:UnicodeCodePoint. vbd=0x00000A
|
||||
pos=0x00000092 len=0x0002 con=1 vbc=3:UnicodeCodePoint. vbd=0x00000A
|
||||
pos=0x00000094 len=0x0001 con=0 vbc=2:String........... vbd=0x000013
|
||||
pos=0x00000095 len=0x0001 con=0 vbc=0:Filler........... vbd=0x000000
|
||||
pos=0x00000096 len=0x0001 con=0 vbc=1:Structure........ vbd=0x001042
|
||||
```
|
||||
@@ -674,7 +674,7 @@ JsonThing::parse_string(TokenStream& ts, TokenStream::Result tsr) {
|
||||
return Result("main: internal error: unexpected token");
|
||||
}
|
||||
|
||||
if (!tsr.token.link_next()) {
|
||||
if (!tsr.token.continued()) {
|
||||
break;
|
||||
}
|
||||
tsr = ts.next();
|
||||
|
||||
@@ -979,15 +979,15 @@ handle_token(wuffs_base__token t, bool start_of_token_chain) {
|
||||
return "main: internal error: unexpected string-token conversion";
|
||||
}
|
||||
|
||||
if (t.link_next()) {
|
||||
if (t.continued()) {
|
||||
return nullptr;
|
||||
}
|
||||
TRY(write_dst("\"", 1));
|
||||
goto after_value;
|
||||
|
||||
case WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT:
|
||||
if (!t.link_next()) {
|
||||
return "main: internal error: unexpected unlinked token";
|
||||
if (!t.continued()) {
|
||||
return "main: internal error: unexpected non-continued UCP token";
|
||||
}
|
||||
TRY(handle_unicode_code_point(vbd));
|
||||
g_query.incremental_match_code_point(vbd);
|
||||
@@ -1049,12 +1049,12 @@ main1(int argc, char** argv) {
|
||||
|
||||
// Skip filler tokens (e.g. whitespace).
|
||||
if (t.value() == 0) {
|
||||
start_of_token_chain = !t.link_next();
|
||||
start_of_token_chain = !t.continued();
|
||||
continue;
|
||||
}
|
||||
|
||||
const char* z = handle_token(t, start_of_token_chain);
|
||||
start_of_token_chain = !t.link_next();
|
||||
start_of_token_chain = !t.continued();
|
||||
if (z == nullptr) {
|
||||
continue;
|
||||
} else if (z == g_eod) {
|
||||
|
||||
@@ -196,7 +196,7 @@ fuzz_one_token(wuffs_base__token t,
|
||||
|
||||
// After a complete JSON value, update the parity (even/odd count) of the
|
||||
// container.
|
||||
if (!wuffs_base__token__link_next(&t) &&
|
||||
if (!wuffs_base__token__continued(&t) &&
|
||||
(vbc != WUFFS_BASE__TOKEN__VBC__FILLER) &&
|
||||
((vbc != WUFFS_BASE__TOKEN__VBC__STRUCTURE) ||
|
||||
(vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP))) {
|
||||
@@ -368,8 +368,8 @@ fuzz_complex(wuffs_base__io_buffer* full_src, uint32_t hash_24_bits) {
|
||||
|
||||
if (depth != 0) {
|
||||
return "fuzz: internal error: decoded OK but final depth was not zero";
|
||||
} else if (wuffs_base__token__link_next(&final_token)) {
|
||||
return "fuzz: internal error: decoded OK but final token had link_next";
|
||||
} else if (wuffs_base__token__continued(&final_token)) {
|
||||
return "fuzz: internal error: decoded OK but final token was continued";
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@@ -29,8 +29,7 @@ typedef struct {
|
||||
inline int64_t value_base_category() const;
|
||||
inline uint64_t value_minor() const;
|
||||
inline uint64_t value_base_detail() const;
|
||||
inline bool link_prev() const;
|
||||
inline bool link_next() const;
|
||||
inline bool continued() const;
|
||||
inline uint64_t length() const;
|
||||
#endif // __cplusplus
|
||||
|
||||
@@ -47,18 +46,15 @@ wuffs_base__make_token(uint64_t repr) {
|
||||
|
||||
#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF
|
||||
|
||||
#define WUFFS_BASE__TOKEN__VALUE__SHIFT 18
|
||||
#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 18
|
||||
#define WUFFS_BASE__TOKEN__VALUE__SHIFT 17
|
||||
#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 17
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42
|
||||
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 39
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 18
|
||||
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 18
|
||||
#define WUFFS_BASE__TOKEN__LINK__SHIFT 16
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 17
|
||||
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 38
|
||||
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 17
|
||||
#define WUFFS_BASE__TOKEN__CONTINUED__SHIFT 16
|
||||
#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0
|
||||
|
||||
#define WUFFS_BASE__TOKEN__LINK_PREV 0x20000
|
||||
#define WUFFS_BASE__TOKEN__LINK_NEXT 0x10000
|
||||
|
||||
// --------
|
||||
|
||||
#define WUFFS_BASE__TOKEN__VBC__FILLER 0
|
||||
@@ -179,7 +175,7 @@ wuffs_base__token__value_base_category(const wuffs_base__token* t) {
|
||||
|
||||
static inline uint64_t //
|
||||
wuffs_base__token__value_minor(const wuffs_base__token* t) {
|
||||
return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & 0xFFFFFF;
|
||||
return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & 0x1FFFFFF;
|
||||
}
|
||||
|
||||
static inline uint64_t //
|
||||
@@ -188,13 +184,8 @@ wuffs_base__token__value_base_detail(const wuffs_base__token* t) {
|
||||
}
|
||||
|
||||
static inline bool //
|
||||
wuffs_base__token__link_prev(const wuffs_base__token* t) {
|
||||
return t->repr & WUFFS_BASE__TOKEN__LINK_PREV;
|
||||
}
|
||||
|
||||
static inline bool //
|
||||
wuffs_base__token__link_next(const wuffs_base__token* t) {
|
||||
return t->repr & WUFFS_BASE__TOKEN__LINK_NEXT;
|
||||
wuffs_base__token__continued(const wuffs_base__token* t) {
|
||||
return t->repr & 0x10000;
|
||||
}
|
||||
|
||||
static inline uint64_t //
|
||||
@@ -235,13 +226,8 @@ wuffs_base__token::value_base_detail() const {
|
||||
}
|
||||
|
||||
inline bool //
|
||||
wuffs_base__token::link_prev() const {
|
||||
return wuffs_base__token__link_prev(this);
|
||||
}
|
||||
|
||||
inline bool //
|
||||
wuffs_base__token::link_next() const {
|
||||
return wuffs_base__token__link_next(this);
|
||||
wuffs_base__token::continued() const {
|
||||
return wuffs_base__token__continued(this);
|
||||
}
|
||||
|
||||
inline uint64_t //
|
||||
|
||||
@@ -350,7 +350,7 @@ func (g *gen) writeBuiltinTokenWriter(b *buffer, recv *a.Expr, method t.ID, args
|
||||
if err := g.writeExpr(b, args[2].AsArg().Value(), depth); err != nil {
|
||||
return err
|
||||
}
|
||||
b.writes(")) << WUFFS_BASE__TOKEN__LINK__SHIFT) | (((uint64_t)(")
|
||||
b.writes(")) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(")
|
||||
}
|
||||
|
||||
if err := g.writeExpr(b, args[3].AsArg().Value(), depth); err != nil {
|
||||
|
||||
@@ -382,9 +382,9 @@ const baseTokenPrivateH = "" +
|
||||
""
|
||||
|
||||
const baseTokenPublicH = "" +
|
||||
"// ---------------- Tokens\n\n// wuffs_base__token is an element of a byte stream's tokenization.\n//\n// See https://github.com/google/wuffs/blob/master/doc/note/tokens.md\ntypedef struct {\n uint64_t repr;\n\n#ifdef __cplusplus\n inline int64_t value() const;\n inline int64_t value_extension() const;\n inline int64_t value_major() const;\n inline int64_t value_base_category() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_detail() const;\n inline bool link_prev() const;\n inline bool link_next() const;\n inline uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\n " +
|
||||
"// ---------------- Tokens\n\n// wuffs_base__token is an element of a byte stream's tokenization.\n//\n// See https://github.com/google/wuffs/blob/master/doc/note/tokens.md\ntypedef struct {\n uint64_t repr;\n\n#ifdef __cplusplus\n inline int64_t value() const;\n inline int64_t value_extension() const;\n inline int64_t value_major() const;\n inline int64_t value_base_category() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_detail() const;\n inline bool continued() const;\n inline uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\n " +
|
||||
"" +
|
||||
"// --------\n\n#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 39\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 18\n#define WUFFS_BASE__TOKEN__LINK__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n#define WUFFS_BASE__TOKEN__LINK_PREV 0x20000\n#define WUFFS_BASE__TOKEN__LINK_NEXT 0x10000\n\n " +
|
||||
"// --------\n\n#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 17\n#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 17\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 17\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 38\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 17\n#define WUFFS_BASE__TOKEN__CONTINUED__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n " +
|
||||
"" +
|
||||
"// --------\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1\n#define WUFFS_BASE__TOKEN__VBC__STRING 2\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 3\n#define WUFFS_BASE__TOKEN__VBC__LITERAL 4\n#define WUFFS_BASE__TOKEN__VBC__NUMBER 5\n\n " +
|
||||
"" +
|
||||
@@ -401,8 +401,8 @@ const baseTokenPublicH = "" +
|
||||
"ttle-endian or text. For binary formats, the\n// token length discriminates e.g. u16 little-endian vs u32 little-endian.\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_BIG_ENDIAN 0x00100\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_LITTLE_ENDIAN 0x00200\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_TEXT 0x00400\n\n" +
|
||||
"" +
|
||||
"// --------\n\n// wuffs_base__token__value returns the token's high 46 bits, sign-extended. A\n// negative value means an extended token, non-negative means a simple token.\nstatic inline int64_t //\nwuffs_base__token__value(const wuffs_base__token* t) {\n return ((int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE__SHIFT;\n}\n\n// wuffs_base__token__value_extension returns a negative value if the token was\n// not an extended token.\nstatic inline int64_t //\nwuffs_base__token__value_extension(const wuffs_base__token* t) {\n return (~(int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT;\n}\n\n// wuffs_base__token__value_major returns a negative value if the token was not\n// a simple token.\nstatic inline int64_t //\nwuffs_base__token__value_major(const wuffs_base__token* t) {\n return ((int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT;\n}\n\n// wuffs_base__token__value_base_category returns a negative value if the token\n// was not a simple token.\nstatic inline int64_t //\nwuffs_base__token__value_base_cat" +
|
||||
"egory(const wuffs_base__token* t) {\n return ((int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_minor(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & 0xFFFFFF;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_detail(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) & 0x1FFFFF;\n}\n\nstatic inline bool //\nwuffs_base__token__link_prev(const wuffs_base__token* t) {\n return t->repr & WUFFS_BASE__TOKEN__LINK_PREV;\n}\n\nstatic inline bool //\nwuffs_base__token__link_next(const wuffs_base__token* t) {\n return t->repr & WUFFS_BASE__TOKEN__LINK_NEXT;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__length(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) & 0xFFFF;\n}\n\n#ifdef __cplusplus\n\ninline int64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline int64_t //\nwuffs_base_" +
|
||||
"_token::value_extension() const {\n return wuffs_base__token__value_extension(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_major() const {\n return wuffs_base__token__value_major(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__token__value_base_detail(this);\n}\n\ninline bool //\nwuffs_base__token::link_prev() const {\n return wuffs_base__token__link_prev(this);\n}\n\ninline bool //\nwuffs_base__token::link_next() const {\n return wuffs_base__token__link_next(this);\n}\n\ninline uint64_t //\nwuffs_base__token::length() const {\n return wuffs_base__token__length(this);\n}\n\n#endif // __cplusplus\n\n" +
|
||||
"egory(const wuffs_base__token* t) {\n return ((int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_minor(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & 0x1FFFFFF;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_detail(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) & 0x1FFFFF;\n}\n\nstatic inline bool //\nwuffs_base__token__continued(const wuffs_base__token* t) {\n return t->repr & 0x10000;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__length(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) & 0xFFFF;\n}\n\n#ifdef __cplusplus\n\ninline int64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_extension() const {\n return wuffs_base__token__value_extension(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_major() const {\n retu" +
|
||||
"rn wuffs_base__token__value_major(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__token__value_base_detail(this);\n}\n\ninline bool //\nwuffs_base__token::continued() const {\n return wuffs_base__token__continued(this);\n}\n\ninline uint64_t //\nwuffs_base__token::length() const {\n return wuffs_base__token__length(this);\n}\n\n#endif // __cplusplus\n\n" +
|
||||
"" +
|
||||
"// --------\n\ntypedef WUFFS_BASE__SLICE(wuffs_base__token) wuffs_base__slice_token;\n\nstatic inline wuffs_base__slice_token //\nwuffs_base__make_slice_token(wuffs_base__token* ptr, size_t len) {\n wuffs_base__slice_token ret;\n ret.ptr = ptr;\n ret.len = len;\n return ret;\n}\n\n" +
|
||||
"" +
|
||||
|
||||
@@ -332,8 +332,8 @@ var Funcs = []string{
|
||||
// ---- token_writer
|
||||
|
||||
"token_writer.write_simple_token_fast!(" +
|
||||
"value_major: u32[..= 0x1F_FFFF], value_minor: u32[..= 0xFF_FFFF]," +
|
||||
"link: u32[..= 0x3], length: u32[..= 0xFFFF])",
|
||||
"value_major: u32[..= 0x1F_FFFF], value_minor: u32[..= 0x1FF_FFFF]," +
|
||||
"continued: u32[..= 0x1], length: u32[..= 0xFFFF])",
|
||||
|
||||
"token_writer.available() u64",
|
||||
|
||||
|
||||
@@ -1888,8 +1888,7 @@ typedef struct {
|
||||
inline int64_t value_base_category() const;
|
||||
inline uint64_t value_minor() const;
|
||||
inline uint64_t value_base_detail() const;
|
||||
inline bool link_prev() const;
|
||||
inline bool link_next() const;
|
||||
inline bool continued() const;
|
||||
inline uint64_t length() const;
|
||||
#endif // __cplusplus
|
||||
|
||||
@@ -1906,18 +1905,15 @@ wuffs_base__make_token(uint64_t repr) {
|
||||
|
||||
#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF
|
||||
|
||||
#define WUFFS_BASE__TOKEN__VALUE__SHIFT 18
|
||||
#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 18
|
||||
#define WUFFS_BASE__TOKEN__VALUE__SHIFT 17
|
||||
#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 17
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42
|
||||
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 39
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 18
|
||||
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 18
|
||||
#define WUFFS_BASE__TOKEN__LINK__SHIFT 16
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 17
|
||||
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 38
|
||||
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 17
|
||||
#define WUFFS_BASE__TOKEN__CONTINUED__SHIFT 16
|
||||
#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0
|
||||
|
||||
#define WUFFS_BASE__TOKEN__LINK_PREV 0x20000
|
||||
#define WUFFS_BASE__TOKEN__LINK_NEXT 0x10000
|
||||
|
||||
// --------
|
||||
|
||||
#define WUFFS_BASE__TOKEN__VBC__FILLER 0
|
||||
@@ -2038,7 +2034,7 @@ wuffs_base__token__value_base_category(const wuffs_base__token* t) {
|
||||
|
||||
static inline uint64_t //
|
||||
wuffs_base__token__value_minor(const wuffs_base__token* t) {
|
||||
return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & 0xFFFFFF;
|
||||
return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & 0x1FFFFFF;
|
||||
}
|
||||
|
||||
static inline uint64_t //
|
||||
@@ -2047,13 +2043,8 @@ wuffs_base__token__value_base_detail(const wuffs_base__token* t) {
|
||||
}
|
||||
|
||||
static inline bool //
|
||||
wuffs_base__token__link_prev(const wuffs_base__token* t) {
|
||||
return t->repr & WUFFS_BASE__TOKEN__LINK_PREV;
|
||||
}
|
||||
|
||||
static inline bool //
|
||||
wuffs_base__token__link_next(const wuffs_base__token* t) {
|
||||
return t->repr & WUFFS_BASE__TOKEN__LINK_NEXT;
|
||||
wuffs_base__token__continued(const wuffs_base__token* t) {
|
||||
return t->repr & 0x10000;
|
||||
}
|
||||
|
||||
static inline uint64_t //
|
||||
@@ -2094,13 +2085,8 @@ wuffs_base__token::value_base_detail() const {
|
||||
}
|
||||
|
||||
inline bool //
|
||||
wuffs_base__token::link_prev() const {
|
||||
return wuffs_base__token__link_prev(this);
|
||||
}
|
||||
|
||||
inline bool //
|
||||
wuffs_base__token::link_next() const {
|
||||
return wuffs_base__token__link_next(this);
|
||||
wuffs_base__token::continued() const {
|
||||
return wuffs_base__token__continued(this);
|
||||
}
|
||||
|
||||
inline uint64_t //
|
||||
@@ -6134,9 +6120,6 @@ struct wuffs_json__decoder__struct {
|
||||
uint32_t v_expect;
|
||||
uint32_t v_expect_after_value;
|
||||
} s_decode_tokens[1];
|
||||
struct {
|
||||
uint32_t v_link_prev;
|
||||
} s_decode_comment[1];
|
||||
struct {
|
||||
uint32_t v_neg;
|
||||
} s_decode_inf_nan[1];
|
||||
@@ -21233,7 +21216,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
if (v_class == 1) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194323)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
(iop_a_src += 1, wuffs_base__make_empty_struct());
|
||||
label__string_loop_outer__continue:;
|
||||
@@ -21252,7 +21235,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(v_string_length))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -21281,7 +21264,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)((v_string_length + 4)))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -21298,7 +21281,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(65532))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -21311,7 +21294,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(v_string_length))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -21322,7 +21305,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(v_string_length))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -21352,7 +21335,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
(((uint64_t)(
|
||||
(6291456 | ((uint32_t)((v_backslash & 127))))))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__string_loop_outer__continue;
|
||||
} else if (v_backslash != 0) {
|
||||
@@ -21365,7 +21348,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
((uint32_t)(wuffs_json__lut_quirky_backslashes[(
|
||||
v_backslash & 7)])))))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1))
|
||||
<< WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__string_loop_outer__continue;
|
||||
}
|
||||
@@ -21411,7 +21395,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)((6291456 | v_uni4_value)))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1))
|
||||
<< WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(6)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__string_loop_outer__continue;
|
||||
} else if (v_uni4_value >= 56320) {
|
||||
@@ -21424,8 +21409,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(6356989))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3))
|
||||
<< WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1))
|
||||
<< WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(6))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__string_loop_outer__continue;
|
||||
@@ -21481,7 +21466,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
(((uint64_t)((6291456 | v_uni4_high_surrogate |
|
||||
v_uni4_value)))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1))
|
||||
<< WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(12))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__string_loop_outer__continue;
|
||||
@@ -21498,7 +21484,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(6356989))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1))
|
||||
<< WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(6)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__string_loop_outer__continue;
|
||||
}
|
||||
@@ -21562,7 +21549,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)((6291456 | (v_uni8_value & 2097151))))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1))
|
||||
<< WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(10)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__string_loop_outer__continue;
|
||||
} else if (self->private_impl
|
||||
@@ -21571,7 +21559,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(6356989))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1))
|
||||
<< WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(10)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__string_loop_outer__continue;
|
||||
}
|
||||
@@ -21619,7 +21608,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194432))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(v_backslash_x_length))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__string_loop_outer__continue;
|
||||
@@ -21633,7 +21622,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1))
|
||||
<< WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(v_string_length))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -21647,7 +21637,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(6356989))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1))
|
||||
<< WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(1))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
(iop_a_src += 1, wuffs_base__make_empty_struct());
|
||||
@@ -21674,7 +21665,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1))
|
||||
<< WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)((v_string_length + 2)))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -21689,7 +21681,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1))
|
||||
<< WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(v_string_length))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -21703,7 +21696,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(6356989))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1))
|
||||
<< WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(1))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
(iop_a_src += 1, wuffs_base__make_empty_struct());
|
||||
@@ -21734,7 +21728,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1))
|
||||
<< WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)((v_string_length + 3)))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -21750,7 +21745,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1))
|
||||
<< WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(v_string_length))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -21764,7 +21760,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(6356989))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1))
|
||||
<< WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(1))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
(iop_a_src += 1, wuffs_base__make_empty_struct());
|
||||
@@ -21795,7 +21792,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1))
|
||||
<< WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)((v_string_length + 4)))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -21810,7 +21808,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194337))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(v_string_length))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_string_length = 0;
|
||||
@@ -21824,7 +21822,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)((6291456 | ((uint32_t)((v_char & 127))))))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
(iop_a_src += 1, wuffs_base__make_empty_struct());
|
||||
goto label__string_loop_outer__continue;
|
||||
@@ -21837,7 +21835,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(6356989))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
(iop_a_src += 1, wuffs_base__make_empty_struct());
|
||||
goto label__string_loop_outer__continue;
|
||||
@@ -21869,7 +21867,6 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(4194323))
|
||||
<< WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
goto label__2__break;
|
||||
}
|
||||
@@ -22566,7 +22563,6 @@ wuffs_json__decoder__decode_comment(wuffs_json__decoder* self,
|
||||
|
||||
uint8_t v_c = 0;
|
||||
uint16_t v_c2 = 0;
|
||||
uint32_t v_link_prev = 0;
|
||||
uint32_t v_length = 0;
|
||||
|
||||
wuffs_base__token* iop_a_dst = NULL;
|
||||
@@ -22594,9 +22590,6 @@ wuffs_json__decoder__decode_comment(wuffs_json__decoder* self,
|
||||
}
|
||||
|
||||
uint32_t coro_susp_point = self->private_impl.p_decode_comment[0];
|
||||
if (coro_susp_point) {
|
||||
v_link_prev = self->private_data.s_decode_comment[0].v_link_prev;
|
||||
}
|
||||
switch (coro_susp_point) {
|
||||
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0;
|
||||
|
||||
@@ -22633,10 +22626,8 @@ wuffs_json__decoder__decode_comment(wuffs_json__decoder* self,
|
||||
if (v_length > 0) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)((1 | v_link_prev)))
|
||||
<< WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(v_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_link_prev = 2;
|
||||
}
|
||||
if (a_src && a_src->meta.closed) {
|
||||
status = wuffs_base__make_status(wuffs_json__error__bad_input);
|
||||
@@ -22653,7 +22644,6 @@ wuffs_json__decoder__decode_comment(wuffs_json__decoder* self,
|
||||
(iop_a_src += 2, wuffs_base__make_empty_struct());
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(v_link_prev)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)((v_length + 2)))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
status = wuffs_base__make_status(NULL);
|
||||
@@ -22663,12 +22653,10 @@ wuffs_json__decoder__decode_comment(wuffs_json__decoder* self,
|
||||
if (v_length >= 65533) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)((1 | v_link_prev)))
|
||||
<< WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)((v_length + 1)))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_length = 0;
|
||||
v_link_prev = 2;
|
||||
goto label__comment_block__continue;
|
||||
}
|
||||
v_length += 1;
|
||||
@@ -22691,10 +22679,8 @@ wuffs_json__decoder__decode_comment(wuffs_json__decoder* self,
|
||||
if (v_length > 0) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)((1 | v_link_prev)))
|
||||
<< WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)(v_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_link_prev = 2;
|
||||
}
|
||||
if (a_src && a_src->meta.closed) {
|
||||
status = wuffs_base__make_status(wuffs_json__error__bad_input);
|
||||
@@ -22711,7 +22697,6 @@ wuffs_json__decoder__decode_comment(wuffs_json__decoder* self,
|
||||
(iop_a_src += 1, wuffs_base__make_empty_struct());
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)(v_link_prev)) << WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)((v_length + 1)))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
status = wuffs_base__make_status(NULL);
|
||||
@@ -22721,12 +22706,10 @@ wuffs_json__decoder__decode_comment(wuffs_json__decoder* self,
|
||||
if (v_length >= 65533) {
|
||||
*iop_a_dst++ = wuffs_base__make_token(
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |
|
||||
(((uint64_t)((1 | v_link_prev)))
|
||||
<< WUFFS_BASE__TOKEN__LINK__SHIFT) |
|
||||
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |
|
||||
(((uint64_t)((v_length + 1)))
|
||||
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
|
||||
v_length = 0;
|
||||
v_link_prev = 2;
|
||||
goto label__comment_line__continue;
|
||||
}
|
||||
v_length += 1;
|
||||
@@ -22745,7 +22728,6 @@ wuffs_json__decoder__decode_comment(wuffs_json__decoder* self,
|
||||
suspend:
|
||||
self->private_impl.p_decode_comment[0] =
|
||||
wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0;
|
||||
self->private_data.s_decode_comment[0].v_link_prev = v_link_prev;
|
||||
|
||||
goto exit;
|
||||
exit:
|
||||
|
||||
@@ -24,16 +24,15 @@
|
||||
// It prints 16 bytes (128 bits) per token, containing big-endian numbers:
|
||||
//
|
||||
// +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
|
||||
// | | | | | | VALUE_EXTENSION |
|
||||
// | POS | LEN | LP| LN|EXT|VALUE_MAJOR| VALUE_MINOR |
|
||||
// | | | | | | 0 |VBC| VBD |
|
||||
// | | | | | VALUE_EXTENSION |
|
||||
// | POS | LEN | CON |EXT|VALUE_MAJOR| VALUE_MINOR |
|
||||
// | | | | | 0 |VBC| VBD |
|
||||
// +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
|
||||
//
|
||||
// - POS (4 bytes) is the position: the sum of all previous tokens' lengths,
|
||||
// including elided tokens.
|
||||
// - LEN (2 bytes) is the length.
|
||||
// - LP (1 bytes) is the link_prev bit.
|
||||
// - LN (1 bytes) is the link_next bit
|
||||
// - CON (2 bytes) is the continued bit
|
||||
// - EXT (1 bytes) is 1 for extended and 0 for simple tokens.
|
||||
//
|
||||
// Extended tokens have a VALUE_EXTENSION (7 bytes).
|
||||
@@ -54,8 +53,8 @@
|
||||
// If the input or output is larger than the program's buffers (64 MiB and
|
||||
// 131072 tokens by default), there may be multiple valid tokenizations of any
|
||||
// given input. For example, if a source string "abcde" straddles an I/O
|
||||
// boundary, it may be tokenized as single (no-link) 5-length string or as a
|
||||
// 3-length link_next'ed string followed by a 2-length link_prev'ed string.
|
||||
// boundary, it may be tokenized as single (not continued) 5-length string or
|
||||
// as a 3-length continued string followed by a 2-length string.
|
||||
//
|
||||
// A Wuffs token stream, in general, can support inputs more than 0xFFFF_FFFF
|
||||
// bytes long, but this program can not, as it tracks the tokens' cumulative
|
||||
@@ -202,7 +201,7 @@ parse_flags(int argc, char** argv) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* g_vbc_names[8] = {
|
||||
const char* g_vbc_names[16] = {
|
||||
"0:Filler..........", //
|
||||
"1:Structure.......", //
|
||||
"2:String..........", //
|
||||
@@ -211,6 +210,14 @@ const char* g_vbc_names[8] = {
|
||||
"5:Number..........", //
|
||||
"6:Reserved........", //
|
||||
"7:Reserved........", //
|
||||
"8:Reserved........", //
|
||||
"9:Reserved........", //
|
||||
"A:Reserved........", //
|
||||
"B:Reserved........", //
|
||||
"C:Reserved........", //
|
||||
"D:Reserved........", //
|
||||
"E:Reserved........", //
|
||||
"F:Reserved........", //
|
||||
};
|
||||
|
||||
const int g_base38_decode[38] = {
|
||||
@@ -277,16 +284,15 @@ main1(int argc, char** argv) {
|
||||
uint16_t len = wuffs_base__token__length(t);
|
||||
|
||||
if (g_flags.all_tokens || (wuffs_base__token__value(t) != 0)) {
|
||||
uint8_t lp = wuffs_base__token__link_prev(t) ? 1 : 0;
|
||||
uint8_t ln = wuffs_base__token__link_next(t) ? 1 : 0;
|
||||
uint16_t con = wuffs_base__token__continued(t) ? 1 : 0;
|
||||
int32_t vmajor = wuffs_base__token__value_major(t);
|
||||
uint32_t vminor = wuffs_base__token__value_minor(t);
|
||||
uint8_t vbc = wuffs_base__token__value_base_category(t);
|
||||
uint32_t vbd = wuffs_base__token__value_base_detail(t);
|
||||
|
||||
if (g_flags.human_readable) {
|
||||
printf("pos=0x%08" PRIX32 " len=0x%04" PRIX16 " link=0b%d%d ",
|
||||
(uint32_t)(pos), len, (int)(lp), (int)(ln));
|
||||
printf("pos=0x%08" PRIX32 " len=0x%04" PRIX16 " con=%" PRId16 " ",
|
||||
(uint32_t)(pos), len, con);
|
||||
|
||||
if (vmajor > 0) {
|
||||
uint32_t m = vmajor;
|
||||
@@ -302,7 +308,8 @@ main1(int argc, char** argv) {
|
||||
vmajor, g_base38_decode[m0], g_base38_decode[m1],
|
||||
g_base38_decode[m2], g_base38_decode[m3], vminor);
|
||||
} else if (vmajor == 0) {
|
||||
printf("vbc=%s. vbd=0x%06" PRIX32 "\n", g_vbc_names[vbc & 7], vbd);
|
||||
printf("vbc=%s. vbd=0x%06" PRIX32 "\n", g_vbc_names[vbc & 15],
|
||||
vbd);
|
||||
} else {
|
||||
printf("extended... vextension=0x%012" PRIX64 "\n",
|
||||
wuffs_base__token__value_extension(t));
|
||||
@@ -312,8 +319,7 @@ main1(int argc, char** argv) {
|
||||
uint8_t buf[16];
|
||||
wuffs_base__store_u32be__no_bounds_check(&buf[0x0], (uint32_t)(pos));
|
||||
wuffs_base__store_u16be__no_bounds_check(&buf[0x4], len);
|
||||
wuffs_base__store_u8__no_bounds_check(&buf[0x0006], lp);
|
||||
wuffs_base__store_u8__no_bounds_check(&buf[0x0007], ln);
|
||||
wuffs_base__store_u16be__no_bounds_check(&buf[0x6], con);
|
||||
if (vmajor > 0) {
|
||||
wuffs_base__store_u32be__no_bounds_check(&buf[0x8], vmajor);
|
||||
wuffs_base__store_u32be__no_bounds_check(&buf[0xC], vminor);
|
||||
|
||||
+65
-70
@@ -188,7 +188,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0,
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: whitespace_length)
|
||||
whitespace_length = 0
|
||||
}
|
||||
@@ -211,7 +211,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0,
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: 0xFFFF)
|
||||
whitespace_length = 0
|
||||
continue.outer
|
||||
@@ -224,7 +224,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0,
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: whitespace_length)
|
||||
whitespace_length = 0
|
||||
if args.dst.available() <= 0 {
|
||||
@@ -250,7 +250,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x40_0013,
|
||||
link: 0x1,
|
||||
continued: 1,
|
||||
length: 1)
|
||||
args.src.skip32_fast!(actual: 1, worst_case: 1)
|
||||
|
||||
@@ -269,7 +269,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x40_0021,
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: string_length)
|
||||
string_length = 0
|
||||
}
|
||||
@@ -299,7 +299,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x40_0021,
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: string_length + 4)
|
||||
string_length = 0
|
||||
continue.string_loop_outer
|
||||
@@ -316,7 +316,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x40_0021,
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: 0xFFFC)
|
||||
string_length = 0
|
||||
continue.string_loop_outer
|
||||
@@ -329,7 +329,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x40_0021,
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: string_length)
|
||||
string_length = 0
|
||||
}
|
||||
@@ -340,7 +340,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x40_0021,
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: string_length)
|
||||
string_length = 0
|
||||
if args.dst.available() <= 0 {
|
||||
@@ -365,7 +365,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x60_0000 | ((backslash & 0x7F) as base.u32),
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: 2)
|
||||
continue.string_loop_outer
|
||||
|
||||
@@ -375,7 +375,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x60_0000 | (lut_quirky_backslashes[backslash & 7] as base.u32),
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: 2)
|
||||
continue.string_loop_outer
|
||||
}
|
||||
@@ -419,7 +419,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x60_0000 | uni4_value,
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: 6)
|
||||
continue.string_loop_outer
|
||||
|
||||
@@ -440,7 +440,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x60_FFFD,
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: 6)
|
||||
continue.string_loop_outer
|
||||
}
|
||||
@@ -491,7 +491,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x60_0000 | uni4_high_surrogate | uni4_value,
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: 12)
|
||||
continue.string_loop_outer
|
||||
}
|
||||
@@ -505,7 +505,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x60_FFFD,
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: 6)
|
||||
continue.string_loop_outer
|
||||
}
|
||||
@@ -563,7 +563,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x60_0000 | (uni8_value & 0x1F_FFFF),
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: 10)
|
||||
continue.string_loop_outer
|
||||
} else if this.quirk_enabled_replace_invalid_unicode {
|
||||
@@ -571,7 +571,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x60_FFFD,
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: 10)
|
||||
continue.string_loop_outer
|
||||
}
|
||||
@@ -618,7 +618,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x40_0080,
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: backslash_x_length)
|
||||
continue.string_loop_outer
|
||||
// -------- END backslash-x
|
||||
@@ -632,7 +632,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x40_0021,
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: string_length)
|
||||
string_length = 0
|
||||
if args.dst.available() <= 0 {
|
||||
@@ -644,7 +644,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x60_FFFD,
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: 1)
|
||||
args.src.skip32_fast!(actual: 1, worst_case: 1)
|
||||
continue.string_loop_outer
|
||||
@@ -665,7 +665,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x40_0021,
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: string_length + 2)
|
||||
string_length = 0
|
||||
continue.string_loop_outer
|
||||
@@ -680,7 +680,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x40_0021,
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: string_length)
|
||||
string_length = 0
|
||||
if args.dst.available() <= 0 {
|
||||
@@ -692,7 +692,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x60_FFFD,
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: 1)
|
||||
args.src.skip32_fast!(actual: 1, worst_case: 1)
|
||||
continue.string_loop_outer
|
||||
@@ -717,7 +717,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x40_0021,
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: string_length + 3)
|
||||
string_length = 0
|
||||
continue.string_loop_outer
|
||||
@@ -733,7 +733,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x40_0021,
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: string_length)
|
||||
string_length = 0
|
||||
if args.dst.available() <= 0 {
|
||||
@@ -745,7 +745,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x60_FFFD,
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: 1)
|
||||
args.src.skip32_fast!(actual: 1, worst_case: 1)
|
||||
continue.string_loop_outer
|
||||
@@ -769,7 +769,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x40_0021,
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: string_length + 4)
|
||||
string_length = 0
|
||||
continue.string_loop_outer
|
||||
@@ -784,7 +784,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x40_0021,
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: string_length)
|
||||
string_length = 0
|
||||
if args.dst.available() <= 0 {
|
||||
@@ -796,7 +796,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x60_0000 | ((char & 0x7F) as base.u32),
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: 1)
|
||||
args.src.skip32_fast!(actual: 1, worst_case: 1)
|
||||
continue.string_loop_outer
|
||||
@@ -807,7 +807,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x60_FFFD,
|
||||
link: 0x3,
|
||||
continued: 1,
|
||||
length: 1)
|
||||
args.src.skip32_fast!(actual: 1, worst_case: 1)
|
||||
continue.string_loop_outer
|
||||
@@ -833,7 +833,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x40_0013,
|
||||
link: 0x2,
|
||||
continued: 0,
|
||||
length: 1)
|
||||
break
|
||||
} endwhile
|
||||
@@ -854,7 +854,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0,
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: 1)
|
||||
// What's valid after a comma depends on whether or not we're in an
|
||||
// array or an object.
|
||||
@@ -879,7 +879,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0,
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: 1)
|
||||
expect = 0x1EB2 // 0x1EB2 is EXPECT_VALUE.
|
||||
continue.outer
|
||||
@@ -900,7 +900,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: vminor,
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: number_length)
|
||||
break
|
||||
}
|
||||
@@ -955,7 +955,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: vminor,
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: 1)
|
||||
expect = 0x1042 // 0x1042 is (EXPECT_CLOSE_CURLY_BRACE | EXPECT_STRING).
|
||||
expect_after_value = 0x1044 // 0x1044 is (EXPECT_CURLY_CLOSE_BRACE | EXPECT_COMMA).
|
||||
@@ -967,7 +967,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x20_1042,
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: 1)
|
||||
break.outer
|
||||
}
|
||||
@@ -978,7 +978,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x20_2042,
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: 1)
|
||||
// 0x1104 is (EXPECT_SQUARE_CLOSE_BRACKET | EXPECT_COMMA).
|
||||
expect = 0x1104
|
||||
@@ -987,7 +987,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x20_4042,
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: 1)
|
||||
// 0x1044 is (EXPECT_CURLY_CLOSE_BRACE | EXPECT_COMMA).
|
||||
expect = 0x1044
|
||||
@@ -1016,7 +1016,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: vminor,
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: 1)
|
||||
expect = 0x1FB2 // 0x1FB2 is (EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_VALUE).
|
||||
expect_after_value = 0x1104 // 0x1104 is (EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA).
|
||||
@@ -1028,7 +1028,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x20_1022,
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: 1)
|
||||
break.outer
|
||||
}
|
||||
@@ -1039,7 +1039,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x20_2022,
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: 1)
|
||||
// 0x1104 is (EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA).
|
||||
expect = 0x1104
|
||||
@@ -1048,7 +1048,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x20_4022,
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: 1)
|
||||
// 0x1044 is (EXPECT_CLOSE_CURLY_BRACE | EXPECT_COMMA).
|
||||
expect = 0x1044
|
||||
@@ -1062,7 +1062,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x80_0004,
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: 5)
|
||||
if args.src.available() < 5 {
|
||||
return "#internal error: inconsistent I/O"
|
||||
@@ -1080,7 +1080,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x80_0008,
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: 4)
|
||||
if args.src.available() < 4 {
|
||||
return "#internal error: inconsistent I/O"
|
||||
@@ -1098,7 +1098,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x80_0002,
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: 4)
|
||||
if args.src.available() < 4 {
|
||||
return "#internal error: inconsistent I/O"
|
||||
@@ -1339,7 +1339,7 @@ pri func decoder.decode_leading?(dst: base.token_writer, src: base.io_reader) {
|
||||
this.allow_leading_ars = false
|
||||
args.src.skip32_fast!(actual: 1, worst_case: 1)
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0, value_minor: 0, link: 0x0, length: 1)
|
||||
value_major: 0, value_minor: 0, continued: 0, length: 1)
|
||||
continue
|
||||
} else if (c == 0xEF) and this.allow_leading_ubom {
|
||||
if args.src.available() < 3 {
|
||||
@@ -1354,7 +1354,7 @@ pri func decoder.decode_leading?(dst: base.token_writer, src: base.io_reader) {
|
||||
this.allow_leading_ubom = false
|
||||
args.src.skip32_fast!(actual: 3, worst_case: 3)
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0, value_minor: 0, link: 0x0, length: 3)
|
||||
value_major: 0, value_minor: 0, continued: 0, length: 3)
|
||||
continue
|
||||
}
|
||||
}
|
||||
@@ -1363,10 +1363,9 @@ pri func decoder.decode_leading?(dst: base.token_writer, src: base.io_reader) {
|
||||
}
|
||||
|
||||
pri func decoder.decode_comment?(dst: base.token_writer, src: base.io_reader) {
|
||||
var c : base.u8
|
||||
var c2 : base.u16
|
||||
var link_prev : base.u32[..= 0x2]
|
||||
var length : base.u32[..= 0xFFFD]
|
||||
var c : base.u8
|
||||
var c2 : base.u16
|
||||
var length : base.u32[..= 0xFFFD]
|
||||
|
||||
while (args.dst.available() <= 0) or (args.src.available() <= 1),
|
||||
post args.dst.available() > 0,
|
||||
@@ -1402,9 +1401,8 @@ pri func decoder.decode_comment?(dst: base.token_writer, src: base.io_reader) {
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x2,
|
||||
link: 0x1 | link_prev,
|
||||
continued: 1,
|
||||
length: length)
|
||||
link_prev = 0x2
|
||||
}
|
||||
if args.src.is_closed() {
|
||||
return "#bad input"
|
||||
@@ -1420,7 +1418,7 @@ pri func decoder.decode_comment?(dst: base.token_writer, src: base.io_reader) {
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x2,
|
||||
link: link_prev,
|
||||
continued: 0,
|
||||
length: length + 2)
|
||||
return ok
|
||||
}
|
||||
@@ -1430,10 +1428,9 @@ pri func decoder.decode_comment?(dst: base.token_writer, src: base.io_reader) {
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x2,
|
||||
link: 0x1 | link_prev,
|
||||
continued: 1,
|
||||
length: length + 1)
|
||||
length = 0
|
||||
link_prev = 0x2
|
||||
continue.comment_block
|
||||
}
|
||||
length += 1
|
||||
@@ -1459,9 +1456,8 @@ pri func decoder.decode_comment?(dst: base.token_writer, src: base.io_reader) {
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x1,
|
||||
link: 0x1 | link_prev,
|
||||
continued: 1,
|
||||
length: length)
|
||||
link_prev = 0x2
|
||||
}
|
||||
if args.src.is_closed() {
|
||||
return "#bad input"
|
||||
@@ -1477,7 +1473,7 @@ pri func decoder.decode_comment?(dst: base.token_writer, src: base.io_reader) {
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x1,
|
||||
link: link_prev,
|
||||
continued: 0,
|
||||
length: length + 1)
|
||||
return ok
|
||||
}
|
||||
@@ -1487,10 +1483,9 @@ pri func decoder.decode_comment?(dst: base.token_writer, src: base.io_reader) {
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0x1,
|
||||
link: 0x1 | link_prev,
|
||||
continued: 1,
|
||||
length: length + 1)
|
||||
length = 0
|
||||
link_prev = 0x2
|
||||
continue.comment_line
|
||||
}
|
||||
length += 1
|
||||
@@ -1527,7 +1522,7 @@ pri func decoder.decode_inf_nan?(dst: base.token_writer, src: base.io_reader) {
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0xA0_0020,
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: 8)
|
||||
args.src.skip32_fast!(actual: 8, worst_case: 8)
|
||||
return ok
|
||||
@@ -1539,7 +1534,7 @@ pri func decoder.decode_inf_nan?(dst: base.token_writer, src: base.io_reader) {
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0xA0_0020,
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: 3)
|
||||
args.src.skip32_fast!(actual: 3, worst_case: 3)
|
||||
return ok
|
||||
@@ -1548,7 +1543,7 @@ pri func decoder.decode_inf_nan?(dst: base.token_writer, src: base.io_reader) {
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0xA0_0080,
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: 3)
|
||||
args.src.skip32_fast!(actual: 3, worst_case: 3)
|
||||
return ok
|
||||
@@ -1575,7 +1570,7 @@ pri func decoder.decode_inf_nan?(dst: base.token_writer, src: base.io_reader) {
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0xA0_0000 | ((0x20 as base.u32) >> neg),
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: 9)
|
||||
args.src.skip32_fast!(actual: 9, worst_case: 9)
|
||||
return ok
|
||||
@@ -1587,7 +1582,7 @@ pri func decoder.decode_inf_nan?(dst: base.token_writer, src: base.io_reader) {
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0xA0_0000 | ((0x20 as base.u32) >> neg),
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: 4)
|
||||
args.src.skip32_fast!(actual: 4, worst_case: 4)
|
||||
return ok
|
||||
@@ -1596,7 +1591,7 @@ pri func decoder.decode_inf_nan?(dst: base.token_writer, src: base.io_reader) {
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: 0xA0_0000 | ((0x80 as base.u32) >> neg),
|
||||
link: 0x0,
|
||||
continued: 0,
|
||||
length: 4)
|
||||
args.src.skip32_fast!(actual: 4, worst_case: 4)
|
||||
return ok
|
||||
@@ -1623,7 +1618,7 @@ pri func decoder.decode_trailing_new_line?(dst: base.token_writer, src: base.io_
|
||||
if args.src.available() <= 0 {
|
||||
if whitespace_length > 0 {
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0, value_minor: 0, link: 0x0, length: whitespace_length)
|
||||
value_major: 0, value_minor: 0, continued: 0, length: whitespace_length)
|
||||
whitespace_length = 0
|
||||
}
|
||||
if args.src.is_closed() {
|
||||
@@ -1638,7 +1633,7 @@ pri func decoder.decode_trailing_new_line?(dst: base.token_writer, src: base.io_
|
||||
if lut_classes[c] <> 0x00 { // 0x00 is CLASS_WHITESPACE.
|
||||
if whitespace_length > 0 {
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0, value_minor: 0, link: 0x0, length: whitespace_length)
|
||||
value_major: 0, value_minor: 0, continued: 0, length: whitespace_length)
|
||||
whitespace_length = 0
|
||||
}
|
||||
return "#bad input"
|
||||
@@ -1647,7 +1642,7 @@ pri func decoder.decode_trailing_new_line?(dst: base.token_writer, src: base.io_
|
||||
args.src.skip32_fast!(actual: 1, worst_case: 1)
|
||||
if (whitespace_length >= 0xFFFE) or (c == '\n') {
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0, value_minor: 0, link: 0x0, length: whitespace_length + 1)
|
||||
value_major: 0, value_minor: 0, continued: 0, length: whitespace_length + 1)
|
||||
whitespace_length = 0
|
||||
if c == '\n' {
|
||||
break.outer
|
||||
|
||||
@@ -1136,8 +1136,7 @@ do_test__wuffs_base__token_decoder(wuffs_base__token_decoder* b,
|
||||
uint16_t len = wuffs_base__token__length(t);
|
||||
|
||||
if (wuffs_base__token__value(t) != 0) {
|
||||
uint8_t lp = wuffs_base__token__link_prev(t) ? 1 : 0;
|
||||
uint8_t ln = wuffs_base__token__link_next(t) ? 1 : 0;
|
||||
uint16_t con = wuffs_base__token__continued(t) ? 1 : 0;
|
||||
int32_t vmajor = wuffs_base__token__value_major(t);
|
||||
|
||||
if ((have.data.len - have.meta.wi) < 16) {
|
||||
@@ -1149,8 +1148,7 @@ do_test__wuffs_base__token_decoder(wuffs_base__token_decoder* b,
|
||||
|
||||
wuffs_base__store_u32be__no_bounds_check(ptr + 0x0, (uint32_t)(pos));
|
||||
wuffs_base__store_u16be__no_bounds_check(ptr + 0x4, len);
|
||||
wuffs_base__store_u8__no_bounds_check(ptr + 0x0006, lp);
|
||||
wuffs_base__store_u8__no_bounds_check(ptr + 0x0007, ln);
|
||||
wuffs_base__store_u16be__no_bounds_check(ptr + 0x6, con);
|
||||
if (vmajor > 0) {
|
||||
wuffs_base__store_u32be__no_bounds_check(ptr + 0x8, vmajor);
|
||||
uint32_t vminor = wuffs_base__token__value_minor(t);
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Reference in new issue
Block a user