Add WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_ETC

This commit is contained in:
Nigel Tao committed 2020-03-11 12:06:50 +11:00
1 parent 499ca132f6
commit c3ca77a539
6 files changed
+32 -17

No files matched your search

+13 -6
View File
@@ -169,16 +169,23 @@ wuffs_base__make_token(uint64_t repr) {
// For a source string of "123" or "0x9A", it is valid for a tokenizer to
// return any one of:
// - WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT.
// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED.
// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED.
// - WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT.
// - WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_SIGNED.
// - WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_UNSIGNED.
//
// For a source string of "+123" or "-0x9A", only the first two are valid.
//
// For a source string of "123.", only the first one is valid.
#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00001
#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED 0x00002
#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED 0x00004
#define WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT 0x00001
#define WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_SIGNED 0x00002
#define WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_UNSIGNED 0x00004
// The number 300 might be represented as "\x01\x2C", "\x2C\x01\x00\x00" or
// "300", which are big-endian, little-endian or text. For binary formats, the
// token length discriminates e.g. u16 little-endian vs u32 little-endian.
#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_BIG_ENDIAN 0x00100
#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_LITTLE_ENDIAN 0x00200
#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_TEXT 0x00400
// --------
+2 -1
View File
@@ -365,7 +365,8 @@ const baseTokenPublicH = "" +
"" +
"// --------\n\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__UNDEFINED 0x00001\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__NULL 0x00002\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__FALSE 0x00004\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__TRUE 0x00008\n\n " +
"" +
"// --------\n\n// For a source string of \"123\" or \"0x9A\", it is valid for a tokenizer to\n// return any one of:\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT.\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED.\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED.\n//\n// For a source string of \"+123\" or \"-0x9A\", only the first two are valid.\n//\n// For a source string of \"123.\", only the first one is valid.\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00001\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED 0x00002\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED 0x00004\n\n" +
"// --------\n\n// For a source string of \"123\" or \"0x9A\", it is valid for a tokenizer to\n// return any one of:\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT.\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_SIGNED.\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_UNSIGNED.\n//\n// For a source string of \"+123\" or \"-0x9A\", only the first two are valid.\n//\n// For a source string of \"123.\", only the first one is valid.\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT 0x00001\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_SIGNED 0x00002\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_UNSIGNED 0x00004\n\n// The number 300 might be represented as \"\\x01\\x2C\", \"\\x2C\\x01\\x00\\x00\" or\n// \"300\", which are big-endian, little-endian or text. For binary formats, the\n// token length discriminates e.g. u16 little-endian vs u32 little-endian.\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_BIG_ENDIAN 0x00100\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_LITTLE_END" +
"IAN 0x00200\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_TEXT 0x00400\n\n" +
"" +
"// --------\n\nstatic inline uint64_t //\nwuffs_base__token__value(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_major(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_minor(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MINOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_category(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_detail(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_BASE_DETA" +
"IL__MASK;\n}\n\nstatic inline bool //\nwuffs_base__token__link_prev(const wuffs_base__token* t) {\n return t->repr & WUFFS_BASE__TOKEN__LINK_PREV;\n}\n\nstatic inline bool //\nwuffs_base__token__link_next(const wuffs_base__token* t) {\n return t->repr & WUFFS_BASE__TOKEN__LINK_NEXT;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__length(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) &\n WUFFS_BASE__TOKEN__LENGTH__MASK;\n}\n\n#ifdef __cplusplus\n\ninline uint64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_major() const {\n return wuffs_base__token__value_major(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__to" +
+15 -8
View File
@@ -1886,16 +1886,23 @@ wuffs_base__make_token(uint64_t repr) {
// For a source string of "123" or "0x9A", it is valid for a tokenizer to
// return any one of:
// - WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT.
// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED.
// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED.
// - WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT.
// - WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_SIGNED.
// - WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_UNSIGNED.
//
// For a source string of "+123" or "-0x9A", only the first two are valid.
//
// For a source string of "123.", only the first one is valid.
#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00001
#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED 0x00002
#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED 0x00004
#define WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT 0x00001
#define WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_SIGNED 0x00002
#define WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_UNSIGNED 0x00004
// The number 300 might be represented as "\x01\x2C", "\x2C\x01\x00\x00" or
// "300", which are big-endian, little-endian or text. For binary formats, the
// token length discriminates e.g. u16 little-endian vs u32 little-endian.
#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_BIG_ENDIAN 0x00100
#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_LITTLE_ENDIAN 0x00200
#define WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_TEXT 0x00400
// --------
@@ -20814,9 +20821,9 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
iop_a_src = a_src->data.ptr + a_src->meta.ri;
}
v_number_status = (v_number_length >> 8);
v_vminor = 10485763;
v_vminor = 10486787;
if ((v_number_length & 128) != 0) {
v_vminor = 10485761;
v_vminor = 10486785;
}
v_number_length = (v_number_length & 127);
if (v_number_status == 0) {
+2 -2
View File
@@ -609,9 +609,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
{
number_length = this.decode_number!(src: args.src)
number_status = number_length >> 8
vminor = 0xA0_0003
vminor = 0xA0_0403
if (number_length & 0x80) <> 0 {
vminor = 0xA0_0001
vminor = 0xA0_0401
}
number_length = number_length & 0x7F
if number_status == 0 {
Binary file not shown.
Binary file not shown.