Define WUFFS_BASE__TOKEN__VBC__ETC macros

This commit is contained in:
Nigel Tao committed 2020-02-23 09:42:02 +11:00
1 parent 01bab82970
commit 9f7a250cf2
4 files changed
+79 -16

No files matched your search

+20 -12
View File
@@ -259,12 +259,15 @@ const char* handle_string(parsed_token pt) {
uint64_t vbc = pt.token.value_base_category();
uint64_t vbd = pt.token.value_base_detail();
if (vbc == 3) {
if (vbc == WUFFS_BASE__TOKEN__VBC__STRING) {
TRY(write_dst(pt.data.ptr, pt.data.len));
if ((vbd & 1) == 0) {
if ((vbd & WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE) == 0) {
break;
}
} else if (vbc != WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT) {
return "main: unexpected token";
} else if (vbd < 0x0020) {
switch (vbd) {
case '\b':
@@ -362,7 +365,8 @@ continue_loop:
uint64_t vbd = pt.token.value_base_detail();
// Handle ']' or '}'.
if ((vbc == 1) && ((vbd & 0x2) != 0)) {
if ((vbc == WUFFS_BASE__TOKEN__VBC__STRUCTURE) &&
((vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP) != 0)) {
if (depth <= 0) {
return "main: internal error: inconsistent depth";
}
@@ -377,9 +381,11 @@ continue_loop:
}
}
TRY(write_dst((vbd & 0x20) ? "]" : "}", 1));
ctx = (vbd & 0x2000) ? context::in_list_after_value
: context::in_dict_after_key;
TRY(write_dst(
(vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST) ? "]" : "}", 1));
ctx = (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST)
? context::in_list_after_value
: context::in_dict_after_key;
goto after_value;
}
@@ -400,18 +406,20 @@ continue_loop:
// Handle the token itself: either a container ('[' or '{') or a simple
// value (number, string or literal).
switch (vbc) {
case 1:
TRY(write_dst((vbd & 0x2000) ? "[" : "{", 1));
case WUFFS_BASE__TOKEN__VBC__STRUCTURE:
TRY(write_dst(
(vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST) ? "[" : "{", 1));
depth++;
ctx = (vbd & 0x2000) ? context::in_list_after_bracket
: context::in_dict_after_brace;
ctx = (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST)
? context::in_list_after_bracket
: context::in_dict_after_brace;
goto continue_loop;
case 2:
case WUFFS_BASE__TOKEN__VBC__NUMBER:
TRY(write_dst(pt.data.ptr, pt.data.len));
goto after_value;
case 3:
case WUFFS_BASE__TOKEN__VBC__STRING:
TRY(handle_string(pt));
goto after_value;
}
+27
View File
@@ -30,6 +30,33 @@
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16
#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0
#define WUFFS_BASE__TOKEN__VBC__FILLER 0
#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1
#define WUFFS_BASE__TOKEN__VBC__NUMBER 2
#define WUFFS_BASE__TOKEN__VBC__STRING 3
#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 4
#define WUFFS_BASE__TOKEN__VBC__BYTES 5
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL 0x00001
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__UNDEFINED 0x00011
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__NULL 0x00021
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__FALSE 0x00041
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__TRUE 0x00081
#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER 0x00002
#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00004
#define WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE 0x00001
#define WUFFS_BASE__TOKEN__VBD__STRING__ALL_ASCII 0x00002
typedef struct {
// Bits 63 .. 40 (24 bits) is the major value.
// Bits 39 .. 16 (24 bits) is the minor value.
+5 -4
View File
@@ -282,10 +282,11 @@ const baseTokenPrivateH = "" +
""
const baseTokenPublicH = "" +
"// ---------------- Tokens\n\n#define WUFFS_BASE__TOKEN__VALUE__MASK 0xFFFFFFFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x7FFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF\n#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 16\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 40\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 16\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 37\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\ntypedef struct {\n // Bits 63 .. 40 (24 bits) is the major value.\n // Bits 39 .. 16 (24 bits) is the minor value.\n // Bits 15 .. 0 (16 bits) is the length.\n uint64_t repr;\n\n#ifdef __cplusplus\n inline uint64_t value() const;\n inline uint64_t value_major() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_category() const;\n inline uin" +
"t64_t value_base_detail() const;\n inline uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_major(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_minor(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MINOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_category(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT) &\n WUFFS_BASE__TOKEN" +
"__VALUE_BASE_CATEGORY__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_detail(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__length(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) &\n WUFFS_BASE__TOKEN__LENGTH__MASK;\n}\n\n#ifdef __cplusplus\n\ninline uint64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_major() const {\n return wuffs_base__token__value_major(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__token__value_base_detail(this);\n}" +
"\n\ninline uint64_t //\nwuffs_base__token::length() const {\n return wuffs_base__token__length(this);\n}\n\n#endif // __cplusplus\n\n" +
"// ---------------- Tokens\n\n#define WUFFS_BASE__TOKEN__VALUE__MASK 0xFFFFFFFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x7FFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF\n#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 16\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 40\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 16\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 37\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1\n#define WUFFS_BASE__TOKEN__VBC__NUMBER 2\n#define WUFFS_BASE__TOKEN__VBC__STRING 3\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 4\n#define WUFFS_BASE__TOKEN__VBC__BYTES 5\n\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x" +
"00002\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000\n\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL 0x00001\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__UNDEFINED 0x00011\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__NULL 0x00021\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__FALSE 0x00041\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__TRUE 0x00081\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER 0x00002\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00004\n\n#define WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRING__ALL_ASCII 0x00002\n\ntypedef struct {\n // Bits 63 .. 40 (24 bits) is the major value.\n // Bits 39 .. 16 (24 bits) is the minor v" +
"alue.\n // Bits 15 .. 0 (16 bits) is the length.\n uint64_t repr;\n\n#ifdef __cplusplus\n inline uint64_t value() const;\n inline uint64_t value_major() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_category() const;\n inline uint64_t value_base_detail() const;\n inline uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_major(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_minor(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_M" +
"INOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MINOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_category(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_detail(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__length(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) &\n WUFFS_BASE__TOKEN__LENGTH__MASK;\n}\n\n#ifdef __cplusplus\n\ninline uint64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_major() const {\n return wuffs_base__token__value_major(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(t" +
"his);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__token__value_base_detail(this);\n}\n\ninline uint64_t //\nwuffs_base__token::length() const {\n return wuffs_base__token__length(this);\n}\n\n#endif // __cplusplus\n\n" +
"" +
"// --------\n\ntypedef WUFFS_BASE__SLICE(wuffs_base__token) wuffs_base__slice_token;\n\nstatic inline wuffs_base__slice_token //\nwuffs_base__make_slice_token(wuffs_base__token* ptr, size_t len) {\n wuffs_base__slice_token ret;\n ret.ptr = ptr;\n ret.len = len;\n return ret;\n}\n\n" +
"" +
+27
View File
@@ -1492,6 +1492,33 @@ wuffs_base__io_buffer::writer_io_position() const {
#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16
#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0
#define WUFFS_BASE__TOKEN__VBC__FILLER 0
#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1
#define WUFFS_BASE__TOKEN__VBC__NUMBER 2
#define WUFFS_BASE__TOKEN__VBC__STRING 3
#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 4
#define WUFFS_BASE__TOKEN__VBC__BYTES 5
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000
#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL 0x00001
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__UNDEFINED 0x00011
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__NULL 0x00021
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__FALSE 0x00041
#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__TRUE 0x00081
#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER 0x00002
#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00004
#define WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE 0x00001
#define WUFFS_BASE__TOKEN__VBD__STRING__ALL_ASCII 0x00002
typedef struct {
// Bits 63 .. 40 (24 bits) is the major value.
// Bits 39 .. 16 (24 bits) is the minor value.