// After editing this file, run "go generate" in the parent directory. // Copyright 2020 The Wuffs Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // https://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // ---------------- Tokens typedef struct { // The repr's 64 bits are divided as: // // +-----+-------------+-------+-------------+-----+-----+-----------+ // | 1 | 21 | 3 | 21 | 1 | 1 | 16 | // +-----+-------------+-------+-------------+-----+-----+-----------+ // [..................value..................] LP LN length // [..0..|.value_major.|.....value_minor.....] // [..0..|.........VBC.........|.....VBD.....] // // The broad divisions are: // - Bits 63 .. 18 (46 bits) is the value. // - Bits 17 .. 16 ( 2 bits) is LP and LN (link_prev and link_next). // - Bits 15 .. 0 (16 bits) is the length. // // The value bits can be sub-divided in multiple ways: // - Bits 63 .. 63 ( 1 bits) is reserved (a zero bit). // - Bits 62 .. 42 (21 bits) is the value_major. // - Bits 41 .. 18 (24 bits) is the value_minor. // - Bits 62 .. 39 (24 bits) is the VBC (value_base_category). // - Bits 38 .. 18 (21 bits) is the VBD (value_base_detail). // // The value_major is a 21-bit [Base38](doc/note/base38-and-fourcc.md) value. // If all of its bits are zero (special cased for Wuffs' built-in "base" // package) then the value_minor is further sub-divided: // - Bits 41 .. 39 ( 3 bits) is the VBC (value_base_category). // - Bits 38 .. 18 (21 bits) is the VBD (value_base_detail). // // The high 46 bits (bits 63 .. 18) only have VBC and VBD semantics when the // high 22 bits (the value_major) are all zero. An equivalent test is that // the high 25 bits (the notional VBC) has a numerical value less than 8. // // At 21 bits, the VBD can hold every valid Unicode code point. // // If value_major is non-zero then value_minor has whatever arbitrary meaning // the tokenizer's package assigns to it. // // Multiple consecutive tokens can form a larger conceptual unit. For // example, an "abc\tz" string is a single higher level concept but at the // lower level, it could consist of multiple tokens: the quotes '"', the // ASCII texts "abc" and "z" and the backslash-escaped tab '\t'. The LP and // LN (link_prev and link_next) bits denote tokens that are part of a // multi-token chain: // - LP means that this token is not the first (there is a previous token). // - LN means that this token is not the last (there is a next token). // // In particular, a stand-alone token will have both link bits set to zero. uint64_t repr; #ifdef __cplusplus inline uint64_t value() const; inline uint64_t value_major() const; inline uint64_t value_minor() const; inline uint64_t value_base_category() const; inline uint64_t value_base_detail() const; inline bool link_prev() const; inline bool link_next() const; inline uint64_t length() const; #endif // __cplusplus } wuffs_base__token; static inline wuffs_base__token // wuffs_base__make_token(uint64_t repr) { wuffs_base__token ret; ret.repr = repr; return ret; } // -------- #define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF #define WUFFS_BASE__TOKEN__VALUE__MASK 0x3FFFFFFFFFFF #define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0x3FFFFF #define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF #define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x1FFFFFF #define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF #define WUFFS_BASE__TOKEN__LINK__MASK 0x3 #define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF #define WUFFS_BASE__TOKEN__VALUE__SHIFT 18 #define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42 #define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 18 #define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 39 #define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 18 #define WUFFS_BASE__TOKEN__LINK__SHIFT 16 #define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0 #define WUFFS_BASE__TOKEN__LINK_PREV 0x20000 #define WUFFS_BASE__TOKEN__LINK_NEXT 0x10000 // -------- #define WUFFS_BASE__TOKEN__VBC__FILLER 0 #define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1 #define WUFFS_BASE__TOKEN__VBC__STRING 2 #define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 3 #define WUFFS_BASE__TOKEN__VBC__LITERAL 4 #define WUFFS_BASE__TOKEN__VBC__NUMBER 5 // -------- #define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00001 #define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00002 // -------- #define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001 #define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002 #define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010 #define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020 #define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040 #define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000 #define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000 #define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000 // -------- // "DEFINITELY_FOO" means that the destination bytes (and also the source // bytes, for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that // the lack of the bit is conservative: it is valid for all-ASCII strings to // have neither DEFINITELY_UTF_8 or DEFINITELY_ASCII bits set. #define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00001 #define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00002 // "CONVERT_D_DST_S_SRC" means that multiples of S source bytes (possibly // padded) produces multiples of D destination bytes. For example, // CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like "\\x23\\x67\\xAB", where // 12 src bytes encode 3 dst bytes. // // When src is the empty string, multiple conversion algorithms are applicable // (so these bits are not necessarily mutually exclusive), all producing the // same empty dst string. #define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00010 #define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00020 #define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00040 #define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00080 #define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x00100 #define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x00200 #define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x00400 // -------- #define WUFFS_BASE__TOKEN__VBD__LITERAL__UNDEFINED 0x00001 #define WUFFS_BASE__TOKEN__VBD__LITERAL__NULL 0x00002 #define WUFFS_BASE__TOKEN__VBD__LITERAL__FALSE 0x00004 #define WUFFS_BASE__TOKEN__VBD__LITERAL__TRUE 0x00008 // -------- // For a source string of "123" or "0x9A", it is valid for a tokenizer to // return any one of: // - WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT. // - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED. // - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED. // // For a source string of "+123" or "-0x9A", only the first two are valid. // // For a source string of "123.", only the first one is valid. #define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00001 #define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED 0x00002 #define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED 0x00004 // -------- static inline uint64_t // wuffs_base__token__value(const wuffs_base__token* t) { return (t->repr >> WUFFS_BASE__TOKEN__VALUE__SHIFT) & WUFFS_BASE__TOKEN__VALUE__MASK; } static inline uint64_t // wuffs_base__token__value_major(const wuffs_base__token* t) { return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT) & WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK; } static inline uint64_t // wuffs_base__token__value_minor(const wuffs_base__token* t) { return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) & WUFFS_BASE__TOKEN__VALUE_MINOR__MASK; } static inline uint64_t // wuffs_base__token__value_base_category(const wuffs_base__token* t) { return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT) & WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK; } static inline uint64_t // wuffs_base__token__value_base_detail(const wuffs_base__token* t) { return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) & WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK; } static inline bool // wuffs_base__token__link_prev(const wuffs_base__token* t) { return t->repr & WUFFS_BASE__TOKEN__LINK_PREV; } static inline bool // wuffs_base__token__link_next(const wuffs_base__token* t) { return t->repr & WUFFS_BASE__TOKEN__LINK_NEXT; } static inline uint64_t // wuffs_base__token__length(const wuffs_base__token* t) { return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) & WUFFS_BASE__TOKEN__LENGTH__MASK; } #ifdef __cplusplus inline uint64_t // wuffs_base__token::value() const { return wuffs_base__token__value(this); } inline uint64_t // wuffs_base__token::value_major() const { return wuffs_base__token__value_major(this); } inline uint64_t // wuffs_base__token::value_minor() const { return wuffs_base__token__value_minor(this); } inline uint64_t // wuffs_base__token::value_base_category() const { return wuffs_base__token__value_base_category(this); } inline uint64_t // wuffs_base__token::value_base_detail() const { return wuffs_base__token__value_base_detail(this); } inline bool // wuffs_base__token::link_prev() const { return wuffs_base__token__link_prev(this); } inline bool // wuffs_base__token::link_next() const { return wuffs_base__token__link_next(this); } inline uint64_t // wuffs_base__token::length() const { return wuffs_base__token__length(this); } #endif // __cplusplus // -------- typedef WUFFS_BASE__SLICE(wuffs_base__token) wuffs_base__slice_token; static inline wuffs_base__slice_token // wuffs_base__make_slice_token(wuffs_base__token* ptr, size_t len) { wuffs_base__slice_token ret; ret.ptr = ptr; ret.len = len; return ret; } // -------- // wuffs_base__token_buffer_meta is the metadata for a // wuffs_base__token_buffer's data. typedef struct { size_t wi; // Write index. Invariant: wi <= len. size_t ri; // Read index. Invariant: ri <= wi. uint64_t pos; // Position of the buffer start relative to the stream start. bool closed; // No further writes are expected. } wuffs_base__token_buffer_meta; // wuffs_base__token_buffer is a 1-dimensional buffer (a pointer and length) // plus additional metadata. // // A value with all fields zero is a valid, empty buffer. typedef struct { wuffs_base__slice_token data; wuffs_base__token_buffer_meta meta; #ifdef __cplusplus inline void compact(); inline uint64_t reader_available() const; inline uint64_t reader_token_position() const; inline uint64_t writer_available() const; inline uint64_t writer_token_position() const; #endif // __cplusplus } wuffs_base__token_buffer; static inline wuffs_base__token_buffer // wuffs_base__make_token_buffer(wuffs_base__slice_token data, wuffs_base__token_buffer_meta meta) { wuffs_base__token_buffer ret; ret.data = data; ret.meta = meta; return ret; } static inline wuffs_base__token_buffer_meta // wuffs_base__make_token_buffer_meta(size_t wi, size_t ri, uint64_t pos, bool closed) { wuffs_base__token_buffer_meta ret; ret.wi = wi; ret.ri = ri; ret.pos = pos; ret.closed = closed; return ret; } static inline wuffs_base__token_buffer // wuffs_base__empty_token_buffer() { wuffs_base__token_buffer ret; ret.data.ptr = NULL; ret.data.len = 0; ret.meta.wi = 0; ret.meta.ri = 0; ret.meta.pos = 0; ret.meta.closed = false; return ret; } static inline wuffs_base__token_buffer_meta // wuffs_base__empty_token_buffer_meta() { wuffs_base__token_buffer_meta ret; ret.wi = 0; ret.ri = 0; ret.pos = 0; ret.closed = false; return ret; } // wuffs_base__token_buffer__compact moves any written but unread tokens to the // start of the buffer. static inline void // wuffs_base__token_buffer__compact(wuffs_base__token_buffer* buf) { if (!buf || (buf->meta.ri == 0)) { return; } buf->meta.pos = wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.ri); size_t n = buf->meta.wi - buf->meta.ri; if (n != 0) { memmove(buf->data.ptr, buf->data.ptr + buf->meta.ri, n * sizeof(wuffs_base__token)); } buf->meta.wi = n; buf->meta.ri = 0; } static inline uint64_t // wuffs_base__token_buffer__reader_available( const wuffs_base__token_buffer* buf) { return buf ? buf->meta.wi - buf->meta.ri : 0; } static inline uint64_t // wuffs_base__token_buffer__reader_token_position( const wuffs_base__token_buffer* buf) { return buf ? wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.ri) : 0; } static inline uint64_t // wuffs_base__token_buffer__writer_available( const wuffs_base__token_buffer* buf) { return buf ? buf->data.len - buf->meta.wi : 0; } static inline uint64_t // wuffs_base__token_buffer__writer_token_position( const wuffs_base__token_buffer* buf) { return buf ? wuffs_base__u64__sat_add(buf->meta.pos, buf->meta.wi) : 0; } #ifdef __cplusplus inline void // wuffs_base__token_buffer::compact() { wuffs_base__token_buffer__compact(this); } inline uint64_t // wuffs_base__token_buffer::reader_available() const { return wuffs_base__token_buffer__reader_available(this); } inline uint64_t // wuffs_base__token_buffer::reader_token_position() const { return wuffs_base__token_buffer__reader_token_position(this); } inline uint64_t // wuffs_base__token_buffer::writer_available() const { return wuffs_base__token_buffer__writer_available(this); } inline uint64_t // wuffs_base__token_buffer::writer_token_position() const { return wuffs_base__token_buffer__writer_token_position(this); } #endif // __cplusplus