diff --git a/doc/note/tokens.md b/doc/note/tokens.md index 8a9bde02..5577f2da 100644 --- a/doc/note/tokens.md +++ b/doc/note/tokens.md @@ -36,25 +36,22 @@ string are treated differently when reconstructing the decoded string: A token is just a `uint64_t`. The broad divisions are: -- Bits 63 .. 18 (46 bits) is the value. -- Bits 17 .. 16 (2 bits) is `LP` and `LN` (`link_prev` and `link_next`). +- Bits 63 .. 17 (47 bits) is the value. +- Bits 16 .. 16 (1 bit) is the `continued` bit. - Bits 15 .. 0 (16 bits) is the length. -The `LP` and `LN` bits denote tokens that are part of a multi-token chain: - -- `LP` means that this token is not the first (there is a previous token). -- `LN` means that this token is not the last (there is a next token). - -A stand-alone token will have both link bits set to zero. +The `continued` bit is whether that the token chain for this token also +contains the next token. The final token in a token chain, including +stand-alone tokens, will have the `continued` bit set to zero. ``` -+-----+-------------+-------+-------------+-----+-----+-----------+ -| 1 | 21 | 3 | 21 | 1 | 1 | 16 | -+-----+-------------+-------+-------------+-----+-----+-----------+ -[..................value..................] LP LN length -[..1..|..........~value_extension.........] -[..0..|.value_major.|.....value_minor.....] -[..0..|......0......|..VBC..|.....VBD.....] ++-----+-------------+-------+---------------+-----+-----------+ +| 1 | 21 | 3 | 22 | 1 | 16 | ++-----+-------------+-------+---------------+-----+-----------+ +[...................value...................] con length +[..1..|...........~value_extension..........] +[..0..|.value_major.|......value_minor......] +[..0..|......0......|..VBC..|......VBD......] ``` The value bits can be sub-divided in multiple ways. First, the high bit: @@ -64,7 +61,7 @@ The value bits can be sub-divided in multiple ways. First, the high bit: ### Extended Tokens -- Bits 62 .. 18 (45 bits) is the bitwise-not (~) of the `value_extension`. +- Bits 62 .. 17 (46 bits) is the bitwise-not (~) of the `value_extension`. Extended tokens are typically part of a multi-token chain whose first token is a simple token that provides the semantics for each `value_extension`. @@ -73,7 +70,7 @@ a simple token that provides the semantics for each `value_extension`. ### Simple Tokens - Bits 62 .. 42 (21 bits) is the `value_major`. -- Bits 41 .. 18 (24 bits) is the `value_minor`. +- Bits 41 .. 17 (25 bits) is the `value_minor`. The `value_major` is a 21-bit [Base38](doc/note/base38-and-fourcc.md) number. For example, an HTML tokenizer might produce a combination of "base" tokens @@ -90,13 +87,13 @@ tokenizer's package assigns to it. A zero `value_major` is reserved for Wuffs' built-in "base" package. The `value_minor` is further sub-divided: - - Bits 41 .. 39 (3 bits) is the `VBC` (`value_base_category`). - - Bits 38 .. 18 (21 bits) is the `VBD` (`value_base_detail`). +- Bits 41 .. 38 (4 bits) is the `VBC` (`value_base_category`). +- Bits 37 .. 17 (21 bits) is the `VBD` (`value_base_detail`). -The high 46 bits (bits 63 .. 18) only have `VBC` and `VBD` semantics when the +The high 47 bits (bits 63 .. 17) only have `VBC` and `VBD` semantics when the high 22 bits (the `extended` and `value_major` parts) are all zero. An -equivalent test is that the high 25 bits (the notional `VBC`), as either an -unsigned integer or a sign-extended integer, is in the range `0 ..= 7`. +equivalent test is that the high 26 bits (the notional `VBC`), as either an +unsigned integer or a sign-extended integer, is in the range `0 ..= 15`. These `VBC`s organize tokens into broad groups, generally applicable (as opposed to being e.g. HTML-specific or JSON-specific). For example, strings and @@ -142,15 +139,17 @@ entire program (not just the Wuffs library) never calls `malloc`. ## Example Token Stream ``` -$ gcc script/print-json-token-debug-format.c && \ -> ./a.out -all-tokens -human-readable < test/data/json-things.formatted.json -pos=0x00000000 len=0x0001 link=0b00 vbc=1:Structure........ vbd=0x004011 -pos=0x00000001 len=0x0005 link=0b00 vbc=0:Filler........... vbd=0x000000 -pos=0x00000006 len=0x0001 link=0b01 vbc=2:String........... vbd=0x000013 -pos=0x00000007 len=0x0002 link=0b11 vbc=2:String........... vbd=0x000021 -pos=0x00000009 len=0x0001 link=0b10 vbc=2:String........... vbd=0x000013 +$ gcc script/print-json-token-debug-format.c -o pjtdf +$ ./pjtdf -all-tokens -human-readable < test/data/json-things.formatted.json +pos=0x00000000 len=0x0001 con=0 vbc=1:Structure........ vbd=0x004011 +pos=0x00000001 len=0x0005 con=0 vbc=0:Filler........... vbd=0x000000 +pos=0x00000006 len=0x0001 con=1 vbc=2:String........... vbd=0x000013 +pos=0x00000007 len=0x0002 con=1 vbc=2:String........... vbd=0x000021 +pos=0x00000009 len=0x0001 con=0 vbc=2:String........... vbd=0x000013 etc -pos=0x00000094 len=0x0001 link=0b10 vbc=2:String........... vbd=0x000013 -pos=0x00000095 len=0x0001 link=0b00 vbc=0:Filler........... vbd=0x000000 -pos=0x00000096 len=0x0001 link=0b00 vbc=1:Structure........ vbd=0x001042 +pos=0x00000090 len=0x0002 con=1 vbc=3:UnicodeCodePoint. vbd=0x00000A +pos=0x00000092 len=0x0002 con=1 vbc=3:UnicodeCodePoint. vbd=0x00000A +pos=0x00000094 len=0x0001 con=0 vbc=2:String........... vbd=0x000013 +pos=0x00000095 len=0x0001 con=0 vbc=0:Filler........... vbd=0x000000 +pos=0x00000096 len=0x0001 con=0 vbc=1:Structure........ vbd=0x001042 ``` diff --git a/example/jsonfindptrs/jsonfindptrs.cc b/example/jsonfindptrs/jsonfindptrs.cc index 2bf74f98..5c5816fb 100644 --- a/example/jsonfindptrs/jsonfindptrs.cc +++ b/example/jsonfindptrs/jsonfindptrs.cc @@ -674,7 +674,7 @@ JsonThing::parse_string(TokenStream& ts, TokenStream::Result tsr) { return Result("main: internal error: unexpected token"); } - if (!tsr.token.link_next()) { + if (!tsr.token.continued()) { break; } tsr = ts.next(); diff --git a/example/jsonptr/jsonptr.cc b/example/jsonptr/jsonptr.cc index e4aa8a7a..7c3dc2c2 100644 --- a/example/jsonptr/jsonptr.cc +++ b/example/jsonptr/jsonptr.cc @@ -979,15 +979,15 @@ handle_token(wuffs_base__token t, bool start_of_token_chain) { return "main: internal error: unexpected string-token conversion"; } - if (t.link_next()) { + if (t.continued()) { return nullptr; } TRY(write_dst("\"", 1)); goto after_value; case WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT: - if (!t.link_next()) { - return "main: internal error: unexpected unlinked token"; + if (!t.continued()) { + return "main: internal error: unexpected non-continued UCP token"; } TRY(handle_unicode_code_point(vbd)); g_query.incremental_match_code_point(vbd); @@ -1049,12 +1049,12 @@ main1(int argc, char** argv) { // Skip filler tokens (e.g. whitespace). if (t.value() == 0) { - start_of_token_chain = !t.link_next(); + start_of_token_chain = !t.continued(); continue; } const char* z = handle_token(t, start_of_token_chain); - start_of_token_chain = !t.link_next(); + start_of_token_chain = !t.continued(); if (z == nullptr) { continue; } else if (z == g_eod) { diff --git a/fuzz/c/std/json_fuzzer.c b/fuzz/c/std/json_fuzzer.c index c3ae16ee..916237c3 100644 --- a/fuzz/c/std/json_fuzzer.c +++ b/fuzz/c/std/json_fuzzer.c @@ -196,7 +196,7 @@ fuzz_one_token(wuffs_base__token t, // After a complete JSON value, update the parity (even/odd count) of the // container. - if (!wuffs_base__token__link_next(&t) && + if (!wuffs_base__token__continued(&t) && (vbc != WUFFS_BASE__TOKEN__VBC__FILLER) && ((vbc != WUFFS_BASE__TOKEN__VBC__STRUCTURE) || (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP))) { @@ -368,8 +368,8 @@ fuzz_complex(wuffs_base__io_buffer* full_src, uint32_t hash_24_bits) { if (depth != 0) { return "fuzz: internal error: decoded OK but final depth was not zero"; - } else if (wuffs_base__token__link_next(&final_token)) { - return "fuzz: internal error: decoded OK but final token had link_next"; + } else if (wuffs_base__token__continued(&final_token)) { + return "fuzz: internal error: decoded OK but final token was continued"; } return NULL; } diff --git a/internal/cgen/base/token-public.h b/internal/cgen/base/token-public.h index aa37f135..eb05a760 100644 --- a/internal/cgen/base/token-public.h +++ b/internal/cgen/base/token-public.h @@ -29,8 +29,7 @@ typedef struct { inline int64_t value_base_category() const; inline uint64_t value_minor() const; inline uint64_t value_base_detail() const; - inline bool link_prev() const; - inline bool link_next() const; + inline bool continued() const; inline uint64_t length() const; #endif // __cplusplus @@ -47,18 +46,15 @@ wuffs_base__make_token(uint64_t repr) { #define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF -#define WUFFS_BASE__TOKEN__VALUE__SHIFT 18 -#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 18 +#define WUFFS_BASE__TOKEN__VALUE__SHIFT 17 +#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 17 #define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42 -#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 39 -#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 18 -#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 18 -#define WUFFS_BASE__TOKEN__LINK__SHIFT 16 +#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 17 +#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 38 +#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 17 +#define WUFFS_BASE__TOKEN__CONTINUED__SHIFT 16 #define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0 -#define WUFFS_BASE__TOKEN__LINK_PREV 0x20000 -#define WUFFS_BASE__TOKEN__LINK_NEXT 0x10000 - // -------- #define WUFFS_BASE__TOKEN__VBC__FILLER 0 @@ -179,7 +175,7 @@ wuffs_base__token__value_base_category(const wuffs_base__token* t) { static inline uint64_t // wuffs_base__token__value_minor(const wuffs_base__token* t) { - return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & 0xFFFFFF; + return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & 0x1FFFFFF; } static inline uint64_t // @@ -188,13 +184,8 @@ wuffs_base__token__value_base_detail(const wuffs_base__token* t) { } static inline bool // -wuffs_base__token__link_prev(const wuffs_base__token* t) { - return t->repr & WUFFS_BASE__TOKEN__LINK_PREV; -} - -static inline bool // -wuffs_base__token__link_next(const wuffs_base__token* t) { - return t->repr & WUFFS_BASE__TOKEN__LINK_NEXT; +wuffs_base__token__continued(const wuffs_base__token* t) { + return t->repr & 0x10000; } static inline uint64_t // @@ -235,13 +226,8 @@ wuffs_base__token::value_base_detail() const { } inline bool // -wuffs_base__token::link_prev() const { - return wuffs_base__token__link_prev(this); -} - -inline bool // -wuffs_base__token::link_next() const { - return wuffs_base__token__link_next(this); +wuffs_base__token::continued() const { + return wuffs_base__token__continued(this); } inline uint64_t // diff --git a/internal/cgen/builtin.go b/internal/cgen/builtin.go index 197c7f36..5eca9123 100644 --- a/internal/cgen/builtin.go +++ b/internal/cgen/builtin.go @@ -350,7 +350,7 @@ func (g *gen) writeBuiltinTokenWriter(b *buffer, recv *a.Expr, method t.ID, args if err := g.writeExpr(b, args[2].AsArg().Value(), depth); err != nil { return err } - b.writes(")) << WUFFS_BASE__TOKEN__LINK__SHIFT) | (((uint64_t)(") + b.writes(")) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(") } if err := g.writeExpr(b, args[3].AsArg().Value(), depth); err != nil { diff --git a/internal/cgen/data.go b/internal/cgen/data.go index 7d5d0e64..82a39737 100644 --- a/internal/cgen/data.go +++ b/internal/cgen/data.go @@ -382,9 +382,9 @@ const baseTokenPrivateH = "" + "" const baseTokenPublicH = "" + - "// ---------------- Tokens\n\n// wuffs_base__token is an element of a byte stream's tokenization.\n//\n// See https://github.com/google/wuffs/blob/master/doc/note/tokens.md\ntypedef struct {\n uint64_t repr;\n\n#ifdef __cplusplus\n inline int64_t value() const;\n inline int64_t value_extension() const;\n inline int64_t value_major() const;\n inline int64_t value_base_category() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_detail() const;\n inline bool link_prev() const;\n inline bool link_next() const;\n inline uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\n " + + "// ---------------- Tokens\n\n// wuffs_base__token is an element of a byte stream's tokenization.\n//\n// See https://github.com/google/wuffs/blob/master/doc/note/tokens.md\ntypedef struct {\n uint64_t repr;\n\n#ifdef __cplusplus\n inline int64_t value() const;\n inline int64_t value_extension() const;\n inline int64_t value_major() const;\n inline int64_t value_base_category() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_detail() const;\n inline bool continued() const;\n inline uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\n " + "" + - "// --------\n\n#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 39\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 18\n#define WUFFS_BASE__TOKEN__LINK__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n#define WUFFS_BASE__TOKEN__LINK_PREV 0x20000\n#define WUFFS_BASE__TOKEN__LINK_NEXT 0x10000\n\n " + + "// --------\n\n#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 17\n#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 17\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 17\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 38\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 17\n#define WUFFS_BASE__TOKEN__CONTINUED__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n " + "" + "// --------\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1\n#define WUFFS_BASE__TOKEN__VBC__STRING 2\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 3\n#define WUFFS_BASE__TOKEN__VBC__LITERAL 4\n#define WUFFS_BASE__TOKEN__VBC__NUMBER 5\n\n " + "" + @@ -401,8 +401,8 @@ const baseTokenPublicH = "" + "ttle-endian or text. For binary formats, the\n// token length discriminates e.g. u16 little-endian vs u32 little-endian.\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_BIG_ENDIAN 0x00100\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_LITTLE_ENDIAN 0x00200\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_TEXT 0x00400\n\n" + "" + "// --------\n\n// wuffs_base__token__value returns the token's high 46 bits, sign-extended. A\n// negative value means an extended token, non-negative means a simple token.\nstatic inline int64_t //\nwuffs_base__token__value(const wuffs_base__token* t) {\n return ((int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE__SHIFT;\n}\n\n// wuffs_base__token__value_extension returns a negative value if the token was\n// not an extended token.\nstatic inline int64_t //\nwuffs_base__token__value_extension(const wuffs_base__token* t) {\n return (~(int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT;\n}\n\n// wuffs_base__token__value_major returns a negative value if the token was not\n// a simple token.\nstatic inline int64_t //\nwuffs_base__token__value_major(const wuffs_base__token* t) {\n return ((int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT;\n}\n\n// wuffs_base__token__value_base_category returns a negative value if the token\n// was not a simple token.\nstatic inline int64_t //\nwuffs_base__token__value_base_cat" + - "egory(const wuffs_base__token* t) {\n return ((int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_minor(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & 0xFFFFFF;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_detail(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) & 0x1FFFFF;\n}\n\nstatic inline bool //\nwuffs_base__token__link_prev(const wuffs_base__token* t) {\n return t->repr & WUFFS_BASE__TOKEN__LINK_PREV;\n}\n\nstatic inline bool //\nwuffs_base__token__link_next(const wuffs_base__token* t) {\n return t->repr & WUFFS_BASE__TOKEN__LINK_NEXT;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__length(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) & 0xFFFF;\n}\n\n#ifdef __cplusplus\n\ninline int64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline int64_t //\nwuffs_base_" + - "_token::value_extension() const {\n return wuffs_base__token__value_extension(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_major() const {\n return wuffs_base__token__value_major(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__token__value_base_detail(this);\n}\n\ninline bool //\nwuffs_base__token::link_prev() const {\n return wuffs_base__token__link_prev(this);\n}\n\ninline bool //\nwuffs_base__token::link_next() const {\n return wuffs_base__token__link_next(this);\n}\n\ninline uint64_t //\nwuffs_base__token::length() const {\n return wuffs_base__token__length(this);\n}\n\n#endif // __cplusplus\n\n" + + "egory(const wuffs_base__token* t) {\n return ((int64_t)(t->repr)) >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_minor(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & 0x1FFFFFF;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_detail(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) & 0x1FFFFF;\n}\n\nstatic inline bool //\nwuffs_base__token__continued(const wuffs_base__token* t) {\n return t->repr & 0x10000;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__length(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) & 0xFFFF;\n}\n\n#ifdef __cplusplus\n\ninline int64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_extension() const {\n return wuffs_base__token__value_extension(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_major() const {\n retu" + + "rn wuffs_base__token__value_major(this);\n}\n\ninline int64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__token__value_base_detail(this);\n}\n\ninline bool //\nwuffs_base__token::continued() const {\n return wuffs_base__token__continued(this);\n}\n\ninline uint64_t //\nwuffs_base__token::length() const {\n return wuffs_base__token__length(this);\n}\n\n#endif // __cplusplus\n\n" + "" + "// --------\n\ntypedef WUFFS_BASE__SLICE(wuffs_base__token) wuffs_base__slice_token;\n\nstatic inline wuffs_base__slice_token //\nwuffs_base__make_slice_token(wuffs_base__token* ptr, size_t len) {\n wuffs_base__slice_token ret;\n ret.ptr = ptr;\n ret.len = len;\n return ret;\n}\n\n" + "" + diff --git a/lang/builtin/builtin.go b/lang/builtin/builtin.go index 1bd52cca..34ba6ec4 100644 --- a/lang/builtin/builtin.go +++ b/lang/builtin/builtin.go @@ -332,8 +332,8 @@ var Funcs = []string{ // ---- token_writer "token_writer.write_simple_token_fast!(" + - "value_major: u32[..= 0x1F_FFFF], value_minor: u32[..= 0xFF_FFFF]," + - "link: u32[..= 0x3], length: u32[..= 0xFFFF])", + "value_major: u32[..= 0x1F_FFFF], value_minor: u32[..= 0x1FF_FFFF]," + + "continued: u32[..= 0x1], length: u32[..= 0xFFFF])", "token_writer.available() u64", diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 9b0b950d..ed2198f5 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -1888,8 +1888,7 @@ typedef struct { inline int64_t value_base_category() const; inline uint64_t value_minor() const; inline uint64_t value_base_detail() const; - inline bool link_prev() const; - inline bool link_next() const; + inline bool continued() const; inline uint64_t length() const; #endif // __cplusplus @@ -1906,18 +1905,15 @@ wuffs_base__make_token(uint64_t repr) { #define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF -#define WUFFS_BASE__TOKEN__VALUE__SHIFT 18 -#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 18 +#define WUFFS_BASE__TOKEN__VALUE__SHIFT 17 +#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 17 #define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42 -#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 39 -#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 18 -#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 18 -#define WUFFS_BASE__TOKEN__LINK__SHIFT 16 +#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 17 +#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 38 +#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 17 +#define WUFFS_BASE__TOKEN__CONTINUED__SHIFT 16 #define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0 -#define WUFFS_BASE__TOKEN__LINK_PREV 0x20000 -#define WUFFS_BASE__TOKEN__LINK_NEXT 0x10000 - // -------- #define WUFFS_BASE__TOKEN__VBC__FILLER 0 @@ -2038,7 +2034,7 @@ wuffs_base__token__value_base_category(const wuffs_base__token* t) { static inline uint64_t // wuffs_base__token__value_minor(const wuffs_base__token* t) { - return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & 0xFFFFFF; + return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & 0x1FFFFFF; } static inline uint64_t // @@ -2047,13 +2043,8 @@ wuffs_base__token__value_base_detail(const wuffs_base__token* t) { } static inline bool // -wuffs_base__token__link_prev(const wuffs_base__token* t) { - return t->repr & WUFFS_BASE__TOKEN__LINK_PREV; -} - -static inline bool // -wuffs_base__token__link_next(const wuffs_base__token* t) { - return t->repr & WUFFS_BASE__TOKEN__LINK_NEXT; +wuffs_base__token__continued(const wuffs_base__token* t) { + return t->repr & 0x10000; } static inline uint64_t // @@ -2094,13 +2085,8 @@ wuffs_base__token::value_base_detail() const { } inline bool // -wuffs_base__token::link_prev() const { - return wuffs_base__token__link_prev(this); -} - -inline bool // -wuffs_base__token::link_next() const { - return wuffs_base__token__link_next(this); +wuffs_base__token::continued() const { + return wuffs_base__token__continued(this); } inline uint64_t // @@ -6134,9 +6120,6 @@ struct wuffs_json__decoder__struct { uint32_t v_expect; uint32_t v_expect_after_value; } s_decode_tokens[1]; - struct { - uint32_t v_link_prev; - } s_decode_comment[1]; struct { uint32_t v_neg; } s_decode_inf_nan[1]; @@ -21233,7 +21216,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, if (v_class == 1) { *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(4194323)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); (iop_a_src += 1, wuffs_base__make_empty_struct()); label__string_loop_outer__continue:; @@ -21252,7 +21235,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -21281,7 +21264,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)((v_string_length + 4))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -21298,7 +21281,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(65532)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -21311,7 +21294,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -21322,7 +21305,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -21352,7 +21335,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, (((uint64_t)( (6291456 | ((uint32_t)((v_backslash & 127)))))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(2)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__string_loop_outer__continue; } else if (v_backslash != 0) { @@ -21365,7 +21348,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, ((uint32_t)(wuffs_json__lut_quirky_backslashes[( v_backslash & 7)]))))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) + << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(2)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__string_loop_outer__continue; } @@ -21411,7 +21395,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)((6291456 | v_uni4_value))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) + << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(6)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__string_loop_outer__continue; } else if (v_uni4_value >= 56320) { @@ -21424,8 +21409,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(6356989)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) - << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) + << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(6)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__string_loop_outer__continue; @@ -21481,7 +21466,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, (((uint64_t)((6291456 | v_uni4_high_surrogate | v_uni4_value))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) + << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(12)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__string_loop_outer__continue; @@ -21498,7 +21484,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(6356989)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) + << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(6)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__string_loop_outer__continue; } @@ -21562,7 +21549,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)((6291456 | (v_uni8_value & 2097151)))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) + << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(10)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__string_loop_outer__continue; } else if (self->private_impl @@ -21571,7 +21559,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(6356989)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) + << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(10)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__string_loop_outer__continue; } @@ -21619,7 +21608,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(4194432)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_backslash_x_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__string_loop_outer__continue; @@ -21633,7 +21622,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) + << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -21647,7 +21637,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(6356989)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) + << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); (iop_a_src += 1, wuffs_base__make_empty_struct()); @@ -21674,7 +21665,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) + << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)((v_string_length + 2))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -21689,7 +21681,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) + << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -21703,7 +21696,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(6356989)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) + << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); (iop_a_src += 1, wuffs_base__make_empty_struct()); @@ -21734,7 +21728,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) + << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)((v_string_length + 3))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -21750,7 +21745,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) + << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -21764,7 +21760,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(6356989)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) + << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); (iop_a_src += 1, wuffs_base__make_empty_struct()); @@ -21795,7 +21792,8 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) + << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)((v_string_length + 4))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -21810,7 +21808,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_string_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_string_length = 0; @@ -21824,7 +21822,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)((6291456 | ((uint32_t)((v_char & 127)))))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); (iop_a_src += 1, wuffs_base__make_empty_struct()); goto label__string_loop_outer__continue; @@ -21837,7 +21835,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(6356989)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); (iop_a_src += 1, wuffs_base__make_empty_struct()); goto label__string_loop_outer__continue; @@ -21869,7 +21867,6 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(4194323)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(2)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__2__break; } @@ -22566,7 +22563,6 @@ wuffs_json__decoder__decode_comment(wuffs_json__decoder* self, uint8_t v_c = 0; uint16_t v_c2 = 0; - uint32_t v_link_prev = 0; uint32_t v_length = 0; wuffs_base__token* iop_a_dst = NULL; @@ -22594,9 +22590,6 @@ wuffs_json__decoder__decode_comment(wuffs_json__decoder* self, } uint32_t coro_susp_point = self->private_impl.p_decode_comment[0]; - if (coro_susp_point) { - v_link_prev = self->private_data.s_decode_comment[0].v_link_prev; - } switch (coro_susp_point) { WUFFS_BASE__COROUTINE_SUSPENSION_POINT_0; @@ -22633,10 +22626,8 @@ wuffs_json__decoder__decode_comment(wuffs_json__decoder* self, if (v_length > 0) { *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(2)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)((1 | v_link_prev))) - << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); - v_link_prev = 2; } if (a_src && a_src->meta.closed) { status = wuffs_base__make_status(wuffs_json__error__bad_input); @@ -22653,7 +22644,6 @@ wuffs_json__decoder__decode_comment(wuffs_json__decoder* self, (iop_a_src += 2, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(2)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(v_link_prev)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | (((uint64_t)((v_length + 2))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); status = wuffs_base__make_status(NULL); @@ -22663,12 +22653,10 @@ wuffs_json__decoder__decode_comment(wuffs_json__decoder* self, if (v_length >= 65533) { *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(2)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)((1 | v_link_prev))) - << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)((v_length + 1))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_length = 0; - v_link_prev = 2; goto label__comment_block__continue; } v_length += 1; @@ -22691,10 +22679,8 @@ wuffs_json__decoder__decode_comment(wuffs_json__decoder* self, if (v_length > 0) { *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)((1 | v_link_prev))) - << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); - v_link_prev = 2; } if (a_src && a_src->meta.closed) { status = wuffs_base__make_status(wuffs_json__error__bad_input); @@ -22711,7 +22697,6 @@ wuffs_json__decoder__decode_comment(wuffs_json__decoder* self, (iop_a_src += 1, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(v_link_prev)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | (((uint64_t)((v_length + 1))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); status = wuffs_base__make_status(NULL); @@ -22721,12 +22706,10 @@ wuffs_json__decoder__decode_comment(wuffs_json__decoder* self, if (v_length >= 65533) { *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)((1 | v_link_prev))) - << WUFFS_BASE__TOKEN__LINK__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)((v_length + 1))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_length = 0; - v_link_prev = 2; goto label__comment_line__continue; } v_length += 1; @@ -22745,7 +22728,6 @@ wuffs_json__decoder__decode_comment(wuffs_json__decoder* self, suspend: self->private_impl.p_decode_comment[0] = wuffs_base__status__is_suspension(&status) ? coro_susp_point : 0; - self->private_data.s_decode_comment[0].v_link_prev = v_link_prev; goto exit; exit: diff --git a/script/print-json-token-debug-format.c b/script/print-json-token-debug-format.c index 89877b98..e8a81bc1 100644 --- a/script/print-json-token-debug-format.c +++ b/script/print-json-token-debug-format.c @@ -24,16 +24,15 @@ // It prints 16 bytes (128 bits) per token, containing big-endian numbers: // // +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ -// | | | | | | VALUE_EXTENSION | -// | POS | LEN | LP| LN|EXT|VALUE_MAJOR| VALUE_MINOR | -// | | | | | | 0 |VBC| VBD | +// | | | | | VALUE_EXTENSION | +// | POS | LEN | CON |EXT|VALUE_MAJOR| VALUE_MINOR | +// | | | | | 0 |VBC| VBD | // +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ // // - POS (4 bytes) is the position: the sum of all previous tokens' lengths, // including elided tokens. // - LEN (2 bytes) is the length. -// - LP (1 bytes) is the link_prev bit. -// - LN (1 bytes) is the link_next bit +// - CON (2 bytes) is the continued bit // - EXT (1 bytes) is 1 for extended and 0 for simple tokens. // // Extended tokens have a VALUE_EXTENSION (7 bytes). @@ -54,8 +53,8 @@ // If the input or output is larger than the program's buffers (64 MiB and // 131072 tokens by default), there may be multiple valid tokenizations of any // given input. For example, if a source string "abcde" straddles an I/O -// boundary, it may be tokenized as single (no-link) 5-length string or as a -// 3-length link_next'ed string followed by a 2-length link_prev'ed string. +// boundary, it may be tokenized as single (not continued) 5-length string or +// as a 3-length continued string followed by a 2-length string. // // A Wuffs token stream, in general, can support inputs more than 0xFFFF_FFFF // bytes long, but this program can not, as it tracks the tokens' cumulative @@ -202,7 +201,7 @@ parse_flags(int argc, char** argv) { return NULL; } -const char* g_vbc_names[8] = { +const char* g_vbc_names[16] = { "0:Filler..........", // "1:Structure.......", // "2:String..........", // @@ -211,6 +210,14 @@ const char* g_vbc_names[8] = { "5:Number..........", // "6:Reserved........", // "7:Reserved........", // + "8:Reserved........", // + "9:Reserved........", // + "A:Reserved........", // + "B:Reserved........", // + "C:Reserved........", // + "D:Reserved........", // + "E:Reserved........", // + "F:Reserved........", // }; const int g_base38_decode[38] = { @@ -277,16 +284,15 @@ main1(int argc, char** argv) { uint16_t len = wuffs_base__token__length(t); if (g_flags.all_tokens || (wuffs_base__token__value(t) != 0)) { - uint8_t lp = wuffs_base__token__link_prev(t) ? 1 : 0; - uint8_t ln = wuffs_base__token__link_next(t) ? 1 : 0; + uint16_t con = wuffs_base__token__continued(t) ? 1 : 0; int32_t vmajor = wuffs_base__token__value_major(t); uint32_t vminor = wuffs_base__token__value_minor(t); uint8_t vbc = wuffs_base__token__value_base_category(t); uint32_t vbd = wuffs_base__token__value_base_detail(t); if (g_flags.human_readable) { - printf("pos=0x%08" PRIX32 " len=0x%04" PRIX16 " link=0b%d%d ", - (uint32_t)(pos), len, (int)(lp), (int)(ln)); + printf("pos=0x%08" PRIX32 " len=0x%04" PRIX16 " con=%" PRId16 " ", + (uint32_t)(pos), len, con); if (vmajor > 0) { uint32_t m = vmajor; @@ -302,7 +308,8 @@ main1(int argc, char** argv) { vmajor, g_base38_decode[m0], g_base38_decode[m1], g_base38_decode[m2], g_base38_decode[m3], vminor); } else if (vmajor == 0) { - printf("vbc=%s. vbd=0x%06" PRIX32 "\n", g_vbc_names[vbc & 7], vbd); + printf("vbc=%s. vbd=0x%06" PRIX32 "\n", g_vbc_names[vbc & 15], + vbd); } else { printf("extended... vextension=0x%012" PRIX64 "\n", wuffs_base__token__value_extension(t)); @@ -312,8 +319,7 @@ main1(int argc, char** argv) { uint8_t buf[16]; wuffs_base__store_u32be__no_bounds_check(&buf[0x0], (uint32_t)(pos)); wuffs_base__store_u16be__no_bounds_check(&buf[0x4], len); - wuffs_base__store_u8__no_bounds_check(&buf[0x0006], lp); - wuffs_base__store_u8__no_bounds_check(&buf[0x0007], ln); + wuffs_base__store_u16be__no_bounds_check(&buf[0x6], con); if (vmajor > 0) { wuffs_base__store_u32be__no_bounds_check(&buf[0x8], vmajor); wuffs_base__store_u32be__no_bounds_check(&buf[0xC], vminor); diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs index c5169d31..44f78e05 100644 --- a/std/json/decode_json.wuffs +++ b/std/json/decode_json.wuffs @@ -188,7 +188,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0, - link: 0x0, + continued: 0, length: whitespace_length) whitespace_length = 0 } @@ -211,7 +211,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0, - link: 0x0, + continued: 0, length: 0xFFFF) whitespace_length = 0 continue.outer @@ -224,7 +224,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0, - link: 0x0, + continued: 0, length: whitespace_length) whitespace_length = 0 if args.dst.available() <= 0 { @@ -250,7 +250,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x40_0013, - link: 0x1, + continued: 1, length: 1) args.src.skip32_fast!(actual: 1, worst_case: 1) @@ -269,7 +269,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x40_0021, - link: 0x3, + continued: 1, length: string_length) string_length = 0 } @@ -299,7 +299,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x40_0021, - link: 0x3, + continued: 1, length: string_length + 4) string_length = 0 continue.string_loop_outer @@ -316,7 +316,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x40_0021, - link: 0x3, + continued: 1, length: 0xFFFC) string_length = 0 continue.string_loop_outer @@ -329,7 +329,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x40_0021, - link: 0x3, + continued: 1, length: string_length) string_length = 0 } @@ -340,7 +340,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x40_0021, - link: 0x3, + continued: 1, length: string_length) string_length = 0 if args.dst.available() <= 0 { @@ -365,7 +365,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x60_0000 | ((backslash & 0x7F) as base.u32), - link: 0x3, + continued: 1, length: 2) continue.string_loop_outer @@ -375,7 +375,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x60_0000 | (lut_quirky_backslashes[backslash & 7] as base.u32), - link: 0x3, + continued: 1, length: 2) continue.string_loop_outer } @@ -419,7 +419,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x60_0000 | uni4_value, - link: 0x3, + continued: 1, length: 6) continue.string_loop_outer @@ -440,7 +440,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x60_FFFD, - link: 0x3, + continued: 1, length: 6) continue.string_loop_outer } @@ -491,7 +491,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x60_0000 | uni4_high_surrogate | uni4_value, - link: 0x3, + continued: 1, length: 12) continue.string_loop_outer } @@ -505,7 +505,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x60_FFFD, - link: 0x3, + continued: 1, length: 6) continue.string_loop_outer } @@ -563,7 +563,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x60_0000 | (uni8_value & 0x1F_FFFF), - link: 0x3, + continued: 1, length: 10) continue.string_loop_outer } else if this.quirk_enabled_replace_invalid_unicode { @@ -571,7 +571,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x60_FFFD, - link: 0x3, + continued: 1, length: 10) continue.string_loop_outer } @@ -618,7 +618,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x40_0080, - link: 0x3, + continued: 1, length: backslash_x_length) continue.string_loop_outer // -------- END backslash-x @@ -632,7 +632,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x40_0021, - link: 0x3, + continued: 1, length: string_length) string_length = 0 if args.dst.available() <= 0 { @@ -644,7 +644,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x60_FFFD, - link: 0x3, + continued: 1, length: 1) args.src.skip32_fast!(actual: 1, worst_case: 1) continue.string_loop_outer @@ -665,7 +665,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x40_0021, - link: 0x3, + continued: 1, length: string_length + 2) string_length = 0 continue.string_loop_outer @@ -680,7 +680,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x40_0021, - link: 0x3, + continued: 1, length: string_length) string_length = 0 if args.dst.available() <= 0 { @@ -692,7 +692,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x60_FFFD, - link: 0x3, + continued: 1, length: 1) args.src.skip32_fast!(actual: 1, worst_case: 1) continue.string_loop_outer @@ -717,7 +717,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x40_0021, - link: 0x3, + continued: 1, length: string_length + 3) string_length = 0 continue.string_loop_outer @@ -733,7 +733,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x40_0021, - link: 0x3, + continued: 1, length: string_length) string_length = 0 if args.dst.available() <= 0 { @@ -745,7 +745,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x60_FFFD, - link: 0x3, + continued: 1, length: 1) args.src.skip32_fast!(actual: 1, worst_case: 1) continue.string_loop_outer @@ -769,7 +769,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x40_0021, - link: 0x3, + continued: 1, length: string_length + 4) string_length = 0 continue.string_loop_outer @@ -784,7 +784,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x40_0021, - link: 0x3, + continued: 1, length: string_length) string_length = 0 if args.dst.available() <= 0 { @@ -796,7 +796,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x60_0000 | ((char & 0x7F) as base.u32), - link: 0x3, + continued: 1, length: 1) args.src.skip32_fast!(actual: 1, worst_case: 1) continue.string_loop_outer @@ -807,7 +807,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x60_FFFD, - link: 0x3, + continued: 1, length: 1) args.src.skip32_fast!(actual: 1, worst_case: 1) continue.string_loop_outer @@ -833,7 +833,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x40_0013, - link: 0x2, + continued: 0, length: 1) break } endwhile @@ -854,7 +854,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0, - link: 0x0, + continued: 0, length: 1) // What's valid after a comma depends on whether or not we're in an // array or an object. @@ -879,7 +879,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0, - link: 0x0, + continued: 0, length: 1) expect = 0x1EB2 // 0x1EB2 is EXPECT_VALUE. continue.outer @@ -900,7 +900,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: vminor, - link: 0x0, + continued: 0, length: number_length) break } @@ -955,7 +955,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: vminor, - link: 0x0, + continued: 0, length: 1) expect = 0x1042 // 0x1042 is (EXPECT_CLOSE_CURLY_BRACE | EXPECT_STRING). expect_after_value = 0x1044 // 0x1044 is (EXPECT_CURLY_CLOSE_BRACE | EXPECT_COMMA). @@ -967,7 +967,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x20_1042, - link: 0x0, + continued: 0, length: 1) break.outer } @@ -978,7 +978,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x20_2042, - link: 0x0, + continued: 0, length: 1) // 0x1104 is (EXPECT_SQUARE_CLOSE_BRACKET | EXPECT_COMMA). expect = 0x1104 @@ -987,7 +987,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x20_4042, - link: 0x0, + continued: 0, length: 1) // 0x1044 is (EXPECT_CURLY_CLOSE_BRACE | EXPECT_COMMA). expect = 0x1044 @@ -1016,7 +1016,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: vminor, - link: 0x0, + continued: 0, length: 1) expect = 0x1FB2 // 0x1FB2 is (EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_VALUE). expect_after_value = 0x1104 // 0x1104 is (EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA). @@ -1028,7 +1028,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x20_1022, - link: 0x0, + continued: 0, length: 1) break.outer } @@ -1039,7 +1039,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x20_2022, - link: 0x0, + continued: 0, length: 1) // 0x1104 is (EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA). expect = 0x1104 @@ -1048,7 +1048,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x20_4022, - link: 0x0, + continued: 0, length: 1) // 0x1044 is (EXPECT_CLOSE_CURLY_BRACE | EXPECT_COMMA). expect = 0x1044 @@ -1062,7 +1062,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x80_0004, - link: 0x0, + continued: 0, length: 5) if args.src.available() < 5 { return "#internal error: inconsistent I/O" @@ -1080,7 +1080,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x80_0008, - link: 0x0, + continued: 0, length: 4) if args.src.available() < 4 { return "#internal error: inconsistent I/O" @@ -1098,7 +1098,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x80_0002, - link: 0x0, + continued: 0, length: 4) if args.src.available() < 4 { return "#internal error: inconsistent I/O" @@ -1339,7 +1339,7 @@ pri func decoder.decode_leading?(dst: base.token_writer, src: base.io_reader) { this.allow_leading_ars = false args.src.skip32_fast!(actual: 1, worst_case: 1) args.dst.write_simple_token_fast!( - value_major: 0, value_minor: 0, link: 0x0, length: 1) + value_major: 0, value_minor: 0, continued: 0, length: 1) continue } else if (c == 0xEF) and this.allow_leading_ubom { if args.src.available() < 3 { @@ -1354,7 +1354,7 @@ pri func decoder.decode_leading?(dst: base.token_writer, src: base.io_reader) { this.allow_leading_ubom = false args.src.skip32_fast!(actual: 3, worst_case: 3) args.dst.write_simple_token_fast!( - value_major: 0, value_minor: 0, link: 0x0, length: 3) + value_major: 0, value_minor: 0, continued: 0, length: 3) continue } } @@ -1363,10 +1363,9 @@ pri func decoder.decode_leading?(dst: base.token_writer, src: base.io_reader) { } pri func decoder.decode_comment?(dst: base.token_writer, src: base.io_reader) { - var c : base.u8 - var c2 : base.u16 - var link_prev : base.u32[..= 0x2] - var length : base.u32[..= 0xFFFD] + var c : base.u8 + var c2 : base.u16 + var length : base.u32[..= 0xFFFD] while (args.dst.available() <= 0) or (args.src.available() <= 1), post args.dst.available() > 0, @@ -1402,9 +1401,8 @@ pri func decoder.decode_comment?(dst: base.token_writer, src: base.io_reader) { args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x2, - link: 0x1 | link_prev, + continued: 1, length: length) - link_prev = 0x2 } if args.src.is_closed() { return "#bad input" @@ -1420,7 +1418,7 @@ pri func decoder.decode_comment?(dst: base.token_writer, src: base.io_reader) { args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x2, - link: link_prev, + continued: 0, length: length + 2) return ok } @@ -1430,10 +1428,9 @@ pri func decoder.decode_comment?(dst: base.token_writer, src: base.io_reader) { args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x2, - link: 0x1 | link_prev, + continued: 1, length: length + 1) length = 0 - link_prev = 0x2 continue.comment_block } length += 1 @@ -1459,9 +1456,8 @@ pri func decoder.decode_comment?(dst: base.token_writer, src: base.io_reader) { args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x1, - link: 0x1 | link_prev, + continued: 1, length: length) - link_prev = 0x2 } if args.src.is_closed() { return "#bad input" @@ -1477,7 +1473,7 @@ pri func decoder.decode_comment?(dst: base.token_writer, src: base.io_reader) { args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x1, - link: link_prev, + continued: 0, length: length + 1) return ok } @@ -1487,10 +1483,9 @@ pri func decoder.decode_comment?(dst: base.token_writer, src: base.io_reader) { args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0x1, - link: 0x1 | link_prev, + continued: 1, length: length + 1) length = 0 - link_prev = 0x2 continue.comment_line } length += 1 @@ -1527,7 +1522,7 @@ pri func decoder.decode_inf_nan?(dst: base.token_writer, src: base.io_reader) { args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0xA0_0020, - link: 0x0, + continued: 0, length: 8) args.src.skip32_fast!(actual: 8, worst_case: 8) return ok @@ -1539,7 +1534,7 @@ pri func decoder.decode_inf_nan?(dst: base.token_writer, src: base.io_reader) { args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0xA0_0020, - link: 0x0, + continued: 0, length: 3) args.src.skip32_fast!(actual: 3, worst_case: 3) return ok @@ -1548,7 +1543,7 @@ pri func decoder.decode_inf_nan?(dst: base.token_writer, src: base.io_reader) { args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0xA0_0080, - link: 0x0, + continued: 0, length: 3) args.src.skip32_fast!(actual: 3, worst_case: 3) return ok @@ -1575,7 +1570,7 @@ pri func decoder.decode_inf_nan?(dst: base.token_writer, src: base.io_reader) { args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0xA0_0000 | ((0x20 as base.u32) >> neg), - link: 0x0, + continued: 0, length: 9) args.src.skip32_fast!(actual: 9, worst_case: 9) return ok @@ -1587,7 +1582,7 @@ pri func decoder.decode_inf_nan?(dst: base.token_writer, src: base.io_reader) { args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0xA0_0000 | ((0x20 as base.u32) >> neg), - link: 0x0, + continued: 0, length: 4) args.src.skip32_fast!(actual: 4, worst_case: 4) return ok @@ -1596,7 +1591,7 @@ pri func decoder.decode_inf_nan?(dst: base.token_writer, src: base.io_reader) { args.dst.write_simple_token_fast!( value_major: 0, value_minor: 0xA0_0000 | ((0x80 as base.u32) >> neg), - link: 0x0, + continued: 0, length: 4) args.src.skip32_fast!(actual: 4, worst_case: 4) return ok @@ -1623,7 +1618,7 @@ pri func decoder.decode_trailing_new_line?(dst: base.token_writer, src: base.io_ if args.src.available() <= 0 { if whitespace_length > 0 { args.dst.write_simple_token_fast!( - value_major: 0, value_minor: 0, link: 0x0, length: whitespace_length) + value_major: 0, value_minor: 0, continued: 0, length: whitespace_length) whitespace_length = 0 } if args.src.is_closed() { @@ -1638,7 +1633,7 @@ pri func decoder.decode_trailing_new_line?(dst: base.token_writer, src: base.io_ if lut_classes[c] <> 0x00 { // 0x00 is CLASS_WHITESPACE. if whitespace_length > 0 { args.dst.write_simple_token_fast!( - value_major: 0, value_minor: 0, link: 0x0, length: whitespace_length) + value_major: 0, value_minor: 0, continued: 0, length: whitespace_length) whitespace_length = 0 } return "#bad input" @@ -1647,7 +1642,7 @@ pri func decoder.decode_trailing_new_line?(dst: base.token_writer, src: base.io_ args.src.skip32_fast!(actual: 1, worst_case: 1) if (whitespace_length >= 0xFFFE) or (c == '\n') { args.dst.write_simple_token_fast!( - value_major: 0, value_minor: 0, link: 0x0, length: whitespace_length + 1) + value_major: 0, value_minor: 0, continued: 0, length: whitespace_length + 1) whitespace_length = 0 if c == '\n' { break.outer diff --git a/test/c/testlib/testlib.c b/test/c/testlib/testlib.c index 8b82fb95..657e44a5 100644 --- a/test/c/testlib/testlib.c +++ b/test/c/testlib/testlib.c @@ -1136,8 +1136,7 @@ do_test__wuffs_base__token_decoder(wuffs_base__token_decoder* b, uint16_t len = wuffs_base__token__length(t); if (wuffs_base__token__value(t) != 0) { - uint8_t lp = wuffs_base__token__link_prev(t) ? 1 : 0; - uint8_t ln = wuffs_base__token__link_next(t) ? 1 : 0; + uint16_t con = wuffs_base__token__continued(t) ? 1 : 0; int32_t vmajor = wuffs_base__token__value_major(t); if ((have.data.len - have.meta.wi) < 16) { @@ -1149,8 +1148,7 @@ do_test__wuffs_base__token_decoder(wuffs_base__token_decoder* b, wuffs_base__store_u32be__no_bounds_check(ptr + 0x0, (uint32_t)(pos)); wuffs_base__store_u16be__no_bounds_check(ptr + 0x4, len); - wuffs_base__store_u8__no_bounds_check(ptr + 0x0006, lp); - wuffs_base__store_u8__no_bounds_check(ptr + 0x0007, ln); + wuffs_base__store_u16be__no_bounds_check(ptr + 0x6, con); if (vmajor > 0) { wuffs_base__store_u32be__no_bounds_check(ptr + 0x8, vmajor); uint32_t vminor = wuffs_base__token__value_minor(t); diff --git a/test/data/australian-abc-local-stations.tokens b/test/data/australian-abc-local-stations.tokens index 4274df83..1750ef54 100644 Binary files a/test/data/australian-abc-local-stations.tokens and b/test/data/australian-abc-local-stations.tokens differ diff --git a/test/data/json-quirks.tokens b/test/data/json-quirks.tokens index d8feab9d..d2518f21 100644 Binary files a/test/data/json-quirks.tokens and b/test/data/json-quirks.tokens differ diff --git a/test/data/json-things.unformatted.tokens b/test/data/json-things.unformatted.tokens index 8fa01379..7bab270e 100644 Binary files a/test/data/json-things.unformatted.tokens and b/test/data/json-things.unformatted.tokens differ