mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-22 00:47:28 +01:00
Add doc/note/tokens.md
This commit is contained in:
1 parent
df3acfbb21
commit
478d1b817c
4 files changed
+173
-121
No files matched your search
@@ -16,63 +16,10 @@
|
||||
|
||||
// ---------------- Tokens
|
||||
|
||||
// wuffs_base__token is an element of a byte stream's tokenization.
|
||||
//
|
||||
// See https://github.com/google/wuffs/blob/master/doc/note/tokens.md
|
||||
typedef struct {
|
||||
// The repr's 64 bits are divided as:
|
||||
//
|
||||
// +-----+-------------+-------+-------------+-----+-----+-----------+
|
||||
// | 1 | 21 | 3 | 21 | 1 | 1 | 16 |
|
||||
// +-----+-------------+-------+-------------+-----+-----+-----------+
|
||||
// [..................value..................] LP LN length
|
||||
// [..1..|..........~value_extension.........]
|
||||
// [..0..|.value_major.|.....value_minor.....]
|
||||
// [..0..|.........VBC.........|.....VBD.....]
|
||||
//
|
||||
// The broad divisions are:
|
||||
// - Bits 63 .. 18 (46 bits) is the value.
|
||||
// - Bits 17 .. 16 ( 2 bits) is LP and LN (link_prev and link_next).
|
||||
// - Bits 15 .. 0 (16 bits) is the length.
|
||||
//
|
||||
// ----
|
||||
//
|
||||
// The value bits can be sub-divided in multiple ways. First, the high bit:
|
||||
// - Bits 63 .. 63 ( 1 bits) is an extended (1) or simple (0) token.
|
||||
//
|
||||
// For extended tokens:
|
||||
// - Bits 62 .. 18 (45 bits) is the bitwise-not (~) of the value_extension.
|
||||
//
|
||||
// For simple tokens:
|
||||
// - Bits 62 .. 42 (21 bits) is the value_major.
|
||||
// - Bits 41 .. 18 (24 bits) is the value_minor.
|
||||
// - Bits 62 .. 39 (24 bits) is the VBC (value_base_category).
|
||||
// - Bits 38 .. 18 (21 bits) is the VBD (value_base_detail).
|
||||
//
|
||||
// The value_major is a 21-bit [Base38](doc/note/base38-and-fourcc.md) value.
|
||||
// If all of its bits are zero (special cased for Wuffs' built-in "base"
|
||||
// package) then the value_minor is further sub-divided:
|
||||
// - Bits 41 .. 39 ( 3 bits) is the VBC (value_base_category).
|
||||
// - Bits 38 .. 18 (21 bits) is the VBD (value_base_detail).
|
||||
//
|
||||
// The high 46 bits (bits 63 .. 18) only have VBC and VBD semantics when the
|
||||
// high 22 bits (the value_major) are all zero. An equivalent test is that
|
||||
// the high 25 bits (the notional VBC) has a value in the range 0 ..= 7.
|
||||
//
|
||||
// At 21 bits, the VBD can hold every valid Unicode code point.
|
||||
//
|
||||
// If value_major is non-zero then value_minor has whatever arbitrary meaning
|
||||
// the tokenizer's package assigns to it.
|
||||
//
|
||||
// ----
|
||||
//
|
||||
// Multiple consecutive tokens can form a larger conceptual unit. For
|
||||
// example, an "abc\tz" string is a single higher level concept but at the
|
||||
// lower level, it could consist of multiple tokens: the quotes '"', the
|
||||
// ASCII texts "abc" and "z" and the backslash-escaped tab '\t'. The LP and
|
||||
// LN (link_prev and link_next) bits denote tokens that are part of a
|
||||
// multi-token chain:
|
||||
// - LP means that this token is not the first (there is a previous token).
|
||||
// - LN means that this token is not the last (there is a next token).
|
||||
//
|
||||
// In particular, a stand-alone token will have both link bits set to zero.
|
||||
uint64_t repr;
|
||||
|
||||
#ifdef __cplusplus
|
||||
@@ -151,6 +98,10 @@ wuffs_base__make_token(uint64_t repr) {
|
||||
// CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like "\\x23\\x67\\xAB", where
|
||||
// 12 src bytes encode 3 dst bytes.
|
||||
//
|
||||
// Post-processing may further transform those D destination bytes (e.g. treat
|
||||
// "\\xFF" as the Unicode code point U+00FF instead of the byte 0xFF), but that
|
||||
// is out of scope of this VBD's semantics.
|
||||
//
|
||||
// When src is the empty string, multiple conversion algorithms are applicable
|
||||
// (so these bits are not necessarily mutually exclusive), all producing the
|
||||
// same empty dst string.
|
||||
|
||||
@@ -382,13 +382,7 @@ const baseTokenPrivateH = "" +
|
||||
""
|
||||
|
||||
const baseTokenPublicH = "" +
|
||||
"// ---------------- Tokens\n\ntypedef struct {\n // The repr's 64 bits are divided as:\n //\n // +-----+-------------+-------+-------------+-----+-----+-----------+\n // | 1 | 21 | 3 | 21 | 1 | 1 | 16 |\n // +-----+-------------+-------+-------------+-----+-----+-----------+\n // [..................value..................] LP LN length\n // [..1..|..........~value_extension.........]\n // [..0..|.value_major.|.....value_minor.....]\n // [..0..|.........VBC.........|.....VBD.....]\n //\n // The broad divisions are:\n // - Bits 63 .. 18 (46 bits) is the value.\n // - Bits 17 .. 16 ( 2 bits) is LP and LN (link_prev and link_next).\n // - Bits 15 .. 0 (16 bits) is the length.\n //\n " +
|
||||
"" +
|
||||
"// ----\n //\n // The value bits can be sub-divided in multiple ways. First, the high bit:\n // - Bits 63 .. 63 ( 1 bits) is an extended (1) or simple (0) token.\n //\n // For extended tokens:\n // - Bits 62 .. 18 (45 bits) is the bitwise-not (~) of the value_extension.\n //\n // For simple tokens:\n // - Bits 62 .. 42 (21 bits) is the value_major.\n // - Bits 41 .. 18 (24 bits) is the value_minor.\n // - Bits 62 .. 39 (24 bits) is the VBC (value_base_category).\n // - Bits 38 .. 18 (21 bits) is the VBD (value_base_detail).\n //\n // The value_major is a 21-bit [Base38](doc/note/base38-and-fourcc.md) value.\n // If all of its bits are zero (special cased for Wuffs' built-in \"base\"\n // package) then the value_minor is further sub-divided:\n // - Bits 41 .. 39 ( 3 bits) is the VBC (value_base_category).\n // - Bits 38 .. 18 (21 bits) is the VBD (value_base_detail).\n //\n // The high 46 bits (bits 63 .. 18) only have VBC and VBD semantics when the\n // high 22 bits (the value_major) are all zero. An eq" +
|
||||
"uivalent test is that\n // the high 25 bits (the notional VBC) has a value in the range 0 ..= 7.\n //\n // At 21 bits, the VBD can hold every valid Unicode code point.\n //\n // If value_major is non-zero then value_minor has whatever arbitrary meaning\n // the tokenizer's package assigns to it.\n //\n " +
|
||||
"" +
|
||||
"// ----\n //\n // Multiple consecutive tokens can form a larger conceptual unit. For\n // example, an \"abc\\tz\" string is a single higher level concept but at the\n // lower level, it could consist of multiple tokens: the quotes '\"', the\n // ASCII texts \"abc\" and \"z\" and the backslash-escaped tab '\\t'. The LP and\n // LN (link_prev and link_next) bits denote tokens that are part of a\n // multi-token chain:\n // - LP means that this token is not the first (there is a previous token).\n // - LN means that this token is not the last (there is a next token).\n //\n // In particular, a stand-alone token will have both link bits set to zero.\n uint64_t repr;\n\n#ifdef __cplusplus\n inline int64_t value() const;\n inline int64_t value_extension() const;\n inline int64_t value_major() const;\n inline int64_t value_base_category() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_detail() const;\n inline bool link_prev() const;\n inline bool link_next() const;\n inline uint64_t length() " +
|
||||
"const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\n " +
|
||||
"// ---------------- Tokens\n\n// wuffs_base__token is an element of a byte stream's tokenization.\n//\n// See https://github.com/google/wuffs/blob/master/doc/note/tokens.md\ntypedef struct {\n uint64_t repr;\n\n#ifdef __cplusplus\n inline int64_t value() const;\n inline int64_t value_extension() const;\n inline int64_t value_major() const;\n inline int64_t value_base_category() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_detail() const;\n inline bool link_prev() const;\n inline bool link_next() const;\n inline uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\n " +
|
||||
"" +
|
||||
"// --------\n\n#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 39\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 18\n#define WUFFS_BASE__TOKEN__LINK__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n#define WUFFS_BASE__TOKEN__LINK_PREV 0x20000\n#define WUFFS_BASE__TOKEN__LINK_NEXT 0x10000\n\n " +
|
||||
"" +
|
||||
@@ -398,8 +392,8 @@ const baseTokenPublicH = "" +
|
||||
"" +
|
||||
"// --------\n\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000\n\n" +
|
||||
"" +
|
||||
"// --------\n\n// \"DEFINITELY_FOO\" means that the destination bytes (and also the source\n// bytes, for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that\n// the lack of the bit is conservative: it is valid for all-ASCII strings to\n// have neither DEFINITELY_UTF_8 or DEFINITELY_ASCII bits set.\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00002\n\n// \"CONVERT_D_DST_S_SRC\" means that multiples of S source bytes (possibly\n// padded) produces multiples of D destination bytes. For example,\n// CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like \"\\\\x23\\\\x67\\\\xAB\", where\n// 12 src bytes encode 3 dst bytes.\n//\n// When src is the empty string, multiple conversion algorithms are applicable\n// (so these bits are not necessarily mutually exclusive), all producing the\n// same empty dst string.\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00020\n#" +
|
||||
"define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00080\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x00100\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x00200\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x00400\n\n " +
|
||||
"// --------\n\n// \"DEFINITELY_FOO\" means that the destination bytes (and also the source\n// bytes, for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that\n// the lack of the bit is conservative: it is valid for all-ASCII strings to\n// have neither DEFINITELY_UTF_8 or DEFINITELY_ASCII bits set.\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00002\n\n// \"CONVERT_D_DST_S_SRC\" means that multiples of S source bytes (possibly\n// padded) produces multiples of D destination bytes. For example,\n// CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like \"\\\\x23\\\\x67\\\\xAB\", where\n// 12 src bytes encode 3 dst bytes.\n//\n// Post-processing may further transform those D destination bytes (e.g. treat\n// \"\\\\xFF\" as the Unicode code point U+00FF instead of the byte 0xFF), but that\n// is out of scope of this VBD's semantics.\n//\n// When src is the empty string, multiple conversion algorithms are applicable\n// (so these bits are not necessarily mutual" +
|
||||
"ly exclusive), all producing the\n// same empty dst string.\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00080\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x00100\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x00200\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x00400\n\n " +
|
||||
"" +
|
||||
"// --------\n\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__UNDEFINED 0x00001\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__NULL 0x00002\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__FALSE 0x00004\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__TRUE 0x00008\n\n " +
|
||||
"" +
|
||||
|
||||
Reference in new issue
Block a user