Tweak wuffs_base__token bitmask assignments

This commit is contained in:
Nigel Tao committed 2020-02-25 12:12:31 +11:00
1 parent 16b0c4652a
commit a9d1488674
4 files changed
+247 -147

No files matched your search

+93 -44
View File
@@ -16,51 +16,23 @@
// ---------------- Tokens
#define WUFFS_BASE__TOKEN__VALUE__MASK 0xFFFFFFFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0xFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x7FFFFFF
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF
#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF
#define WUFFS_BASE__TOKEN__VALUE__SHIFT 16
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 40
#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 16
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 37
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16
#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0
#define WUFFS_BASE__TOKEN__VBC__FILLER 0
#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1
#define WUFFS_BASE__TOKEN__VBC__NUMBER 2
#define WUFFS_BASE__TOKEN__VBC__STRING 3
#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 4
#define WUFFS_BASE__TOKEN__VBC__BYTES 5
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL 0x00001
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__UNDEFINED 0x00011
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__NULL 0x00021
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__FALSE 0x00041
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__TRUE 0x00081
#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER 0x00002
#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00004
#define WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE 0x00001
#define WUFFS_BASE__TOKEN__VBD__STRING__ALL_ASCII 0x00002
typedef struct {
// Bits 63 .. 40 (24 bits) is the major value.
// Bits 39 .. 16 (24 bits) is the minor value.
// Bits 15 .. 0 (16 bits) is the length.
// The repr is divided as:
// - Bits 63 .. 40 (24 bits) is the major value.
// - Bits 39 .. 16 (24 bits) is the minor value.
// - Bits 15 .. 0 (16 bits) is the length.
//
// The major value is a [Base38](doc/note/base38-and-fourcc.md) value. If
// zero (special cased for Wuffs' built-in "base" package) then the minor
// value is further sub-divided:
// - Bits 39 .. 37 ( 3 bits) is the value_base_category.
// - Bits 36 .. 16 (21 bits) is the value_base_detail.
//
// In particular, at 21 bits, the value_base_detail can hold every valid
// Unicode code point.
//
// If the major value is non-zero then the minor value has whatever arbitrary
// meaning the tokenizer's package assigns to it.
uint64_t repr;
#ifdef __cplusplus
@@ -81,6 +53,83 @@ wuffs_base__make_token(uint64_t repr) {
return ret;
}
#define WUFFS_BASE__TOKEN__VALUE__MASK 0xFFFFFFFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0xFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x7FFFFFF
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF
#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF
#define WUFFS_BASE__TOKEN__VALUE__SHIFT 16
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 40
#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 16
#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 37
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16
#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0
#define WUFFS_BASE__TOKEN__VBC__FILLER 0
#define WUFFS_BASE__TOKEN__VBC__STRING 1
#define WUFFS_BASE__TOKEN__VBC__BYTES 2
#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 3
#define WUFFS_BASE__TOKEN__VBC__NUMBER 4
#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 5
// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between
// VBD_FILLER, VBD_STRING and VBD_BYTES.
#define WUFFS_BASE__TOKEN__VBD__FILLER__INCOMPLETE 0x00001
#define WUFFS_BASE__TOKEN__VBD__FILLER__END_OF_CONSECUTIVE_COMMENTS 0x00010
#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00020
#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00040
// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between
// VBD_FILLER, VBD_STRING and VBD_BYTES.
#define WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE 0x00001
#define WUFFS_BASE__TOKEN__VBD__STRING__ALL_ASCII 0x00010
// Bits 0x2, 0x4 and 0x8 are reserved for flags that are common between
// VBD_FILLER, VBD_STRING and VBD_BYTES.
#define WUFFS_BASE__TOKEN__VBD__BYTES__INCOMPLETE 0x00001
// "D_DST_S_SRC" means that it takes S source bytes (possibly padded) to
// produce D destination bytes. For example,
// WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_4_SRC_BACKSLASH_X means that the source
// looks like "\\x23\\x67\\xAB", where 12 src bytes encode 3 dst bytes.
#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_1_SRC_RAW 0x00010
#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_2_SRC_HEX 0x00020
#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_4_SRC_BACKSLASH_X 0x00040
#define WUFFS_BASE__TOKEN__VBD__BYTES__1_DST_6_SRC_BACKSLASH_U 0x00080
#define WUFFS_BASE__TOKEN__VBD__BYTES__3_DST_4_SRC_BASE_64_STD 0x00100
#define WUFFS_BASE__TOKEN__VBD__BYTES__3_DST_4_SRC_BASE_64_URL 0x00200
#define WUFFS_BASE__TOKEN__VBD__BYTES__4_DST_5_SRC_ASCII_85 0x00400
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL 0x00001
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__UNDEFINED 0x00101
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__NULL 0x00201
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__FALSE 0x00401
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__TRUE 0x00801
// For a source string of "123" or "0x9A", it is valid for a tokenizer to
// return any one of:
// - WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT.
// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED.
// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED.
//
// For a source string of "+123" or "-0x9A", only the first two are valid.
//
// For a source string of "123.", only the first one is valid.
#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00002
#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED 0x00004
#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED 0x00008
#define WUFFS_BASE__TOKEN__VBD__UNICODE_CODE_POINT__MAX_INCL 0x10FFFF
static inline uint64_t //
wuffs_base__token__value(const wuffs_base__token* t) {
return (t->repr >> WUFFS_BASE__TOKEN__VALUE__SHIFT) &