From 9f7a250cf2ff2e668d8a8d238f58a54b0853fe91 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sun, 23 Feb 2020 09:42:02 +1100 Subject: [PATCH] Define WUFFS_BASE__TOKEN__VBC__ETC macros --- example/jsonptr/jsonptr.cc | 32 ++++++++++++++++---------- internal/cgen/base/token-public.h | 27 ++++++++++++++++++++++ internal/cgen/data.go | 9 ++++---- release/c/wuffs-unsupported-snapshot.c | 27 ++++++++++++++++++++++ 4 files changed, 79 insertions(+), 16 deletions(-) diff --git a/example/jsonptr/jsonptr.cc b/example/jsonptr/jsonptr.cc index 1599501b..a6ee3553 100644 --- a/example/jsonptr/jsonptr.cc +++ b/example/jsonptr/jsonptr.cc @@ -259,12 +259,15 @@ const char* handle_string(parsed_token pt) { uint64_t vbc = pt.token.value_base_category(); uint64_t vbd = pt.token.value_base_detail(); - if (vbc == 3) { + if (vbc == WUFFS_BASE__TOKEN__VBC__STRING) { TRY(write_dst(pt.data.ptr, pt.data.len)); - if ((vbd & 1) == 0) { + if ((vbd & WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE) == 0) { break; } + } else if (vbc != WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT) { + return "main: unexpected token"; + } else if (vbd < 0x0020) { switch (vbd) { case '\b': @@ -362,7 +365,8 @@ continue_loop: uint64_t vbd = pt.token.value_base_detail(); // Handle ']' or '}'. - if ((vbc == 1) && ((vbd & 0x2) != 0)) { + if ((vbc == WUFFS_BASE__TOKEN__VBC__STRUCTURE) && + ((vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP) != 0)) { if (depth <= 0) { return "main: internal error: inconsistent depth"; } @@ -377,9 +381,11 @@ continue_loop: } } - TRY(write_dst((vbd & 0x20) ? "]" : "}", 1)); - ctx = (vbd & 0x2000) ? context::in_list_after_value - : context::in_dict_after_key; + TRY(write_dst( + (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST) ? "]" : "}", 1)); + ctx = (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST) + ? context::in_list_after_value + : context::in_dict_after_key; goto after_value; } @@ -400,18 +406,20 @@ continue_loop: // Handle the token itself: either a container ('[' or '{') or a simple // value (number, string or literal). switch (vbc) { - case 1: - TRY(write_dst((vbd & 0x2000) ? "[" : "{", 1)); + case WUFFS_BASE__TOKEN__VBC__STRUCTURE: + TRY(write_dst( + (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST) ? "[" : "{", 1)); depth++; - ctx = (vbd & 0x2000) ? context::in_list_after_bracket - : context::in_dict_after_brace; + ctx = (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST) + ? context::in_list_after_bracket + : context::in_dict_after_brace; goto continue_loop; - case 2: + case WUFFS_BASE__TOKEN__VBC__NUMBER: TRY(write_dst(pt.data.ptr, pt.data.len)); goto after_value; - case 3: + case WUFFS_BASE__TOKEN__VBC__STRING: TRY(handle_string(pt)); goto after_value; } diff --git a/internal/cgen/base/token-public.h b/internal/cgen/base/token-public.h index 9a86e1de..efc7dac7 100644 --- a/internal/cgen/base/token-public.h +++ b/internal/cgen/base/token-public.h @@ -30,6 +30,33 @@ #define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16 #define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0 +#define WUFFS_BASE__TOKEN__VBC__FILLER 0 +#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1 +#define WUFFS_BASE__TOKEN__VBC__NUMBER 2 +#define WUFFS_BASE__TOKEN__VBC__STRING 3 +#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 4 +#define WUFFS_BASE__TOKEN__VBC__BYTES 5 + +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000 + +#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL 0x00001 +#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__UNDEFINED 0x00011 +#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__NULL 0x00021 +#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__FALSE 0x00041 +#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__TRUE 0x00081 +#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER 0x00002 +#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00004 + +#define WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE 0x00001 +#define WUFFS_BASE__TOKEN__VBD__STRING__ALL_ASCII 0x00002 + typedef struct { // Bits 63 .. 40 (24 bits) is the major value. // Bits 39 .. 16 (24 bits) is the minor value. diff --git a/internal/cgen/data.go b/internal/cgen/data.go index 27164c19..772e2f70 100644 --- a/internal/cgen/data.go +++ b/internal/cgen/data.go @@ -282,10 +282,11 @@ const baseTokenPrivateH = "" + "" const baseTokenPublicH = "" + - "// ---------------- Tokens\n\n#define WUFFS_BASE__TOKEN__VALUE__MASK 0xFFFFFFFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x7FFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF\n#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 16\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 40\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 16\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 37\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\ntypedef struct {\n // Bits 63 .. 40 (24 bits) is the major value.\n // Bits 39 .. 16 (24 bits) is the minor value.\n // Bits 15 .. 0 (16 bits) is the length.\n uint64_t repr;\n\n#ifdef __cplusplus\n inline uint64_t value() const;\n inline uint64_t value_major() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_category() const;\n inline uin" + - "t64_t value_base_detail() const;\n inline uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_major(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_minor(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MINOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_category(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT) &\n WUFFS_BASE__TOKEN" + - "__VALUE_BASE_CATEGORY__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_detail(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__length(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) &\n WUFFS_BASE__TOKEN__LENGTH__MASK;\n}\n\n#ifdef __cplusplus\n\ninline uint64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_major() const {\n return wuffs_base__token__value_major(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__token__value_base_detail(this);\n}" + - "\n\ninline uint64_t //\nwuffs_base__token::length() const {\n return wuffs_base__token__length(this);\n}\n\n#endif // __cplusplus\n\n" + + "// ---------------- Tokens\n\n#define WUFFS_BASE__TOKEN__VALUE__MASK 0xFFFFFFFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x7FFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF\n#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 16\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 40\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 16\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 37\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1\n#define WUFFS_BASE__TOKEN__VBC__NUMBER 2\n#define WUFFS_BASE__TOKEN__VBC__STRING 3\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 4\n#define WUFFS_BASE__TOKEN__VBC__BYTES 5\n\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x" + + "00002\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000\n\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL 0x00001\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__UNDEFINED 0x00011\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__NULL 0x00021\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__FALSE 0x00041\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__TRUE 0x00081\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER 0x00002\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00004\n\n#define WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRING__ALL_ASCII 0x00002\n\ntypedef struct {\n // Bits 63 .. 40 (24 bits) is the major value.\n // Bits 39 .. 16 (24 bits) is the minor v" + + "alue.\n // Bits 15 .. 0 (16 bits) is the length.\n uint64_t repr;\n\n#ifdef __cplusplus\n inline uint64_t value() const;\n inline uint64_t value_major() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_category() const;\n inline uint64_t value_base_detail() const;\n inline uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_major(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_minor(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_M" + + "INOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MINOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_category(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_detail(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__length(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) &\n WUFFS_BASE__TOKEN__LENGTH__MASK;\n}\n\n#ifdef __cplusplus\n\ninline uint64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_major() const {\n return wuffs_base__token__value_major(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(t" + + "his);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__token__value_base_detail(this);\n}\n\ninline uint64_t //\nwuffs_base__token::length() const {\n return wuffs_base__token__length(this);\n}\n\n#endif // __cplusplus\n\n" + "" + "// --------\n\ntypedef WUFFS_BASE__SLICE(wuffs_base__token) wuffs_base__slice_token;\n\nstatic inline wuffs_base__slice_token //\nwuffs_base__make_slice_token(wuffs_base__token* ptr, size_t len) {\n wuffs_base__slice_token ret;\n ret.ptr = ptr;\n ret.len = len;\n return ret;\n}\n\n" + "" + diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 76c1191a..0c4b0eba 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -1492,6 +1492,33 @@ wuffs_base__io_buffer::writer_io_position() const { #define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 16 #define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0 +#define WUFFS_BASE__TOKEN__VBC__FILLER 0 +#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1 +#define WUFFS_BASE__TOKEN__VBC__NUMBER 2 +#define WUFFS_BASE__TOKEN__VBC__STRING 3 +#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 4 +#define WUFFS_BASE__TOKEN__VBC__BYTES 5 + +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000 + +#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL 0x00001 +#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__UNDEFINED 0x00011 +#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__NULL 0x00021 +#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__FALSE 0x00041 +#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__TRUE 0x00081 +#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER 0x00002 +#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00004 + +#define WUFFS_BASE__TOKEN__VBD__STRING__INCOMPLETE 0x00001 +#define WUFFS_BASE__TOKEN__VBD__STRING__ALL_ASCII 0x00002 + typedef struct { // Bits 63 .. 40 (24 bits) is the major value. // Bits 39 .. 16 (24 bits) is the minor value.