diff --git a/example/jsonptr/jsonptr.cc b/example/jsonptr/jsonptr.cc index c4b28ce6..c4f849b8 100644 --- a/example/jsonptr/jsonptr.cc +++ b/example/jsonptr/jsonptr.cc @@ -333,8 +333,17 @@ handle_token(wuffs_base__token t) { } // Handle the token itself: either a container ('[' or '{') or a simple - // value (number, string or literal). + // value: string (a chain of raw or escaped parts), literal or number. switch (vbc) { + case WUFFS_BASE__TOKEN__VBC__STRUCTURE: + TRY(write_dst( + (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST) ? "[" : "{", 1)); + depth++; + ctx = (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST) + ? context::in_list_after_bracket + : context::in_dict_after_brace; + return nullptr; + case WUFFS_BASE__TOKEN__VBC__STRING: if (!t.link_prev()) { TRY(write_dst("\"", 1)); @@ -358,18 +367,10 @@ handle_token(wuffs_base__token t) { case WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT: return handle_unicode_code_point(vbd); + case WUFFS_BASE__TOKEN__VBC__LITERAL: case WUFFS_BASE__TOKEN__VBC__NUMBER: TRY(write_dst(src.data.ptr + curr_token_end_src_index - len, len)); goto after_value; - - case WUFFS_BASE__TOKEN__VBC__STRUCTURE: - TRY(write_dst( - (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST) ? "[" : "{", 1)); - depth++; - ctx = (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST) - ? context::in_list_after_bracket - : context::in_dict_after_brace; - return nullptr; } // Return an error if we didn't match the (vbc, vbd) pair. diff --git a/internal/cgen/base/token-public.h b/internal/cgen/base/token-public.h index b34f073a..1343ab97 100644 --- a/internal/cgen/base/token-public.h +++ b/internal/cgen/base/token-public.h @@ -109,16 +109,28 @@ wuffs_base__make_token(uint64_t repr) { // -------- #define WUFFS_BASE__TOKEN__VBC__FILLER 0 -#define WUFFS_BASE__TOKEN__VBC__STRING 1 -#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 2 -#define WUFFS_BASE__TOKEN__VBC__NUMBER 3 -#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 4 +#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1 +#define WUFFS_BASE__TOKEN__VBC__STRING 2 +#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 3 +#define WUFFS_BASE__TOKEN__VBC__LITERAL 4 +#define WUFFS_BASE__TOKEN__VBC__NUMBER 5 // -------- #define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00001 #define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00002 + // -------- + +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000 + // -------- // "DEFINITELY_FOO" means that the destination bytes (and also the source @@ -146,11 +158,12 @@ wuffs_base__make_token(uint64_t repr) { // -------- -#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL 0x00001 -#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__UNDEFINED 0x00101 -#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__NULL 0x00201 -#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__FALSE 0x00401 -#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__TRUE 0x00801 +#define WUFFS_BASE__TOKEN__VBD__LITERAL__UNDEFINED 0x00001 +#define WUFFS_BASE__TOKEN__VBD__LITERAL__NULL 0x00002 +#define WUFFS_BASE__TOKEN__VBD__LITERAL__FALSE 0x00004 +#define WUFFS_BASE__TOKEN__VBD__LITERAL__TRUE 0x00008 + + // -------- // For a source string of "123" or "0x9A", it is valid for a tokenizer to // return any one of: @@ -161,20 +174,9 @@ wuffs_base__make_token(uint64_t repr) { // For a source string of "+123" or "-0x9A", only the first two are valid. // // For a source string of "123.", only the first one is valid. -#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00002 -#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED 0x00004 -#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED 0x00008 - - // -------- - -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000 +#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00001 +#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED 0x00002 +#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED 0x00004 // -------- diff --git a/internal/cgen/data.go b/internal/cgen/data.go index fee98742..65769c73 100644 --- a/internal/cgen/data.go +++ b/internal/cgen/data.go @@ -314,16 +314,18 @@ const baseTokenPublicH = "" + "" + "// --------\n\n#define WUFFS_BASE__TOKEN__VALUE__MASK 0x3FFFFFFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0x3FFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x1FFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF\n#define WUFFS_BASE__TOKEN__LINK__MASK 0x3\n#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 39\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 18\n#define WUFFS_BASE__TOKEN__LINK__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n#define WUFFS_BASE__TOKEN__LINK_PREV 0x20000\n#define WUFFS_BASE__TOKEN__LINK_NEXT 0x10000\n\n " + "" + - "// --------\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRING 1\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 2\n#define WUFFS_BASE__TOKEN__VBC__NUMBER 3\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 4\n\n " + + "// --------\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1\n#define WUFFS_BASE__TOKEN__VBC__STRING 2\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 3\n#define WUFFS_BASE__TOKEN__VBC__LITERAL 4\n#define WUFFS_BASE__TOKEN__VBC__NUMBER 5\n\n " + "" + - "// --------\n\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00001\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00002\n\n" + + "// --------\n\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00001\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00002\n\n " + + "" + + "// --------\n\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000\n\n" + "" + "// --------\n\n// \"DEFINITELY_FOO\" means that the destination bytes (and also the source\n// bytes, for 1_DST_1_SRC_COPY) are in the FOO format. Definitely means that\n// the lack of the bit is conservative: it is valid for all-ASCII strings to\n// have neither DEFINITELY_UTF_8 or DEFINITELY_ASCII bits set.\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_UTF_8 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRING__DEFINITELY_ASCII 0x00002\n\n// \"CONVERT_D_DST_S_SRC\" means that multiples of S source bytes (possibly\n// padded) produces multiples of D destination bytes. For example,\n// CONVERT_1_DST_4_SRC_BACKSLASH_X means a source like \"\\\\x23\\\\x67\\\\xAB\", where\n// 12 src bytes encode 3 dst bytes.\n//\n// When src is the empty string, multiple conversion algorithms are applicable\n// (so these bits are not necessarily mutually exclusive), all producing the\n// same empty dst string.\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY 0x00020\n#" + "define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_2_SRC_HEXADECIMAL 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X 0x00080\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_STD 0x00100\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_3_DST_4_SRC_BASE_64_URL 0x00200\n#define WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_4_DST_5_SRC_ASCII_85 0x00400\n\n " + "" + - "// --------\n\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL 0x00001\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__UNDEFINED 0x00101\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__NULL 0x00201\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__FALSE 0x00401\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__TRUE 0x00801\n\n// For a source string of \"123\" or \"0x9A\", it is valid for a tokenizer to\n// return any one of:\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT.\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED.\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED.\n//\n// For a source string of \"+123\" or \"-0x9A\", only the first two are valid.\n//\n// For a source string of \"123.\", only the first one is valid.\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00002\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED 0x00004\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED 0x00008\n\n " + + "// --------\n\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__UNDEFINED 0x00001\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__NULL 0x00002\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__FALSE 0x00004\n#define WUFFS_BASE__TOKEN__VBD__LITERAL__TRUE 0x00008\n\n " + "" + - "// --------\n\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000\n\n" + + "// --------\n\n// For a source string of \"123\" or \"0x9A\", it is valid for a tokenizer to\n// return any one of:\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT.\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED.\n// - WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED.\n//\n// For a source string of \"+123\" or \"-0x9A\", only the first two are valid.\n//\n// For a source string of \"123.\", only the first one is valid.\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00001\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED 0x00002\n#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED 0x00004\n\n" + "" + "// --------\n\nstatic inline uint64_t //\nwuffs_base__token__value(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_major(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_minor(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_MINOR__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_category(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__value_base_detail(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT) &\n WUFFS_BASE__TOKEN__VALUE_BASE_DETA" + "IL__MASK;\n}\n\nstatic inline bool //\nwuffs_base__token__link_prev(const wuffs_base__token* t) {\n return t->repr & WUFFS_BASE__TOKEN__LINK_PREV;\n}\n\nstatic inline bool //\nwuffs_base__token__link_next(const wuffs_base__token* t) {\n return t->repr & WUFFS_BASE__TOKEN__LINK_NEXT;\n}\n\nstatic inline uint64_t //\nwuffs_base__token__length(const wuffs_base__token* t) {\n return (t->repr >> WUFFS_BASE__TOKEN__LENGTH__SHIFT) &\n WUFFS_BASE__TOKEN__LENGTH__MASK;\n}\n\n#ifdef __cplusplus\n\ninline uint64_t //\nwuffs_base__token::value() const {\n return wuffs_base__token__value(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_major() const {\n return wuffs_base__token__value_major(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_minor() const {\n return wuffs_base__token__value_minor(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_category() const {\n return wuffs_base__token__value_base_category(this);\n}\n\ninline uint64_t //\nwuffs_base__token::value_base_detail() const {\n return wuffs_base__to" + diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 75ca1012..87bbe885 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -1825,16 +1825,28 @@ wuffs_base__make_token(uint64_t repr) { // -------- #define WUFFS_BASE__TOKEN__VBC__FILLER 0 -#define WUFFS_BASE__TOKEN__VBC__STRING 1 -#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 2 -#define WUFFS_BASE__TOKEN__VBC__NUMBER 3 -#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 4 +#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1 +#define WUFFS_BASE__TOKEN__VBC__STRING 2 +#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 3 +#define WUFFS_BASE__TOKEN__VBC__LITERAL 4 +#define WUFFS_BASE__TOKEN__VBC__NUMBER 5 // -------- #define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00001 #define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00002 + // -------- + +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000 +#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000 + // -------- // "DEFINITELY_FOO" means that the destination bytes (and also the source @@ -1862,11 +1874,12 @@ wuffs_base__make_token(uint64_t repr) { // -------- -#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL 0x00001 -#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__UNDEFINED 0x00101 -#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__NULL 0x00201 -#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__FALSE 0x00401 -#define WUFFS_BASE__TOKEN__VBD__NUMBER__LITERAL__TRUE 0x00801 +#define WUFFS_BASE__TOKEN__VBD__LITERAL__UNDEFINED 0x00001 +#define WUFFS_BASE__TOKEN__VBD__LITERAL__NULL 0x00002 +#define WUFFS_BASE__TOKEN__VBD__LITERAL__FALSE 0x00004 +#define WUFFS_BASE__TOKEN__VBD__LITERAL__TRUE 0x00008 + + // -------- // For a source string of "123" or "0x9A", it is valid for a tokenizer to // return any one of: @@ -1877,20 +1890,9 @@ wuffs_base__make_token(uint64_t repr) { // For a source string of "+123" or "-0x9A", only the first two are valid. // // For a source string of "123.", only the first one is valid. -#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00002 -#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED 0x00004 -#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED 0x00008 - - // -------- - -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000 -#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000 +#define WUFFS_BASE__TOKEN__VBD__NUMBER__FLOATING_POINT 0x00001 +#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_SIGNED 0x00002 +#define WUFFS_BASE__TOKEN__VBD__NUMBER__INTEGER_UNSIGNED 0x00004 // -------- @@ -19403,7 +19405,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } if (v_class == 1) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(2097171)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(4194323)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); (iop_a_src += 1, wuffs_base__make_empty_struct()); @@ -19421,7 +19423,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) { if (v_string_length > 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(2097185)) + (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | (((uint64_t)(v_string_length)) @@ -19445,7 +19447,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, (iop_a_src += 1, wuffs_base__make_empty_struct()); if (v_string_length >= 65531) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(2097185)) + (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | (((uint64_t)(65532)) @@ -19458,7 +19460,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } else if (v_char == 1) { if (v_string_length != 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(2097185)) + (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | (((uint64_t)(v_string_length)) @@ -19469,7 +19471,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } else if (v_char == 2) { if (v_string_length > 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(2097185)) + (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | (((uint64_t)(v_string_length)) @@ -19499,7 +19501,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, (iop_a_src += 2, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)( - (4194304 | ((uint32_t)((v_backslash & 127)))))) + (6291456 | ((uint32_t)((v_backslash & 127)))))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | (((uint64_t)(2)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); @@ -19544,7 +19546,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } else if ((v_uni4_value < 55296) || (57343 < v_uni4_value)) { (iop_a_src += 6, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)((4194304 | v_uni4_value))) + (((uint64_t)((6291456 | v_uni4_value))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | (((uint64_t)(6)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); @@ -19602,7 +19604,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, v_uni4_value -= 56320; (iop_a_src += 8, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)((4194304 | v_uni4_high_surrogate | + (((uint64_t)((6291456 | v_uni4_high_surrogate | v_uni4_value))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | @@ -19649,7 +19651,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, (iop_a_src += 2, wuffs_base__make_empty_struct()); if (v_string_length >= 65528) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(2097185)) + (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | (((uint64_t)((v_string_length + 2))) @@ -19686,7 +19688,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, (iop_a_src += 3, wuffs_base__make_empty_struct()); if (v_string_length >= 65528) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(2097185)) + (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | (((uint64_t)((v_string_length + 3))) @@ -19724,7 +19726,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, (iop_a_src += 4, wuffs_base__make_empty_struct()); if (v_string_length >= 65528) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(2097185)) + (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | (((uint64_t)((v_string_length + 4))) @@ -19739,7 +19741,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } if (v_string_length > 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(2097185)) + (((uint64_t)(4194337)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(3)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | (((uint64_t)(v_string_length)) @@ -19776,7 +19778,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } (iop_a_src += 1, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(2097171)) + (((uint64_t)(4194323)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(2)) << WUFFS_BASE__TOKEN__LINK__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); @@ -19816,9 +19818,9 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, iop_a_src = a_src->data.ptr + a_src->meta.ri; } v_number_status = (v_number_length >> 8); - v_vminor = 6291460; + v_vminor = 10485763; if ((v_number_length & 128) != 0) { - v_vminor = 6291458; + v_vminor = 10485761; } v_number_length = (v_number_length & 127); if (v_number_status == 0) { @@ -19860,12 +19862,12 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, label__2__break:; goto label__goto_parsed_a_leaf_value__break; } else if (v_class == 5) { - v_vminor = 8405009; + v_vminor = 2113553; if (v_depth == 0) { } else if (0 != (v_expect_after_value & 64)) { - v_vminor = 8405057; + v_vminor = 2113601; } else { - v_vminor = 8405025; + v_vminor = 2113569; } if (v_depth >= 1024) { status = wuffs_base__make_status( @@ -19889,7 +19891,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, (iop_a_src += 1, wuffs_base__make_empty_struct()); if (v_depth <= 1) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(8392770)) + (((uint64_t)(2101314)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__outer__break; @@ -19900,14 +19902,14 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, if (0 == (self->private_data.f_stack[v_stack_byte] & (((uint32_t)(1)) << v_stack_bit))) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(8396866)) + (((uint64_t)(2105410)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_expect = 260; v_expect_after_value = 260; } else { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(8405058)) + (((uint64_t)(2113602)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_expect = 68; @@ -19915,12 +19917,12 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, } goto label__outer__continue; } else if (v_class == 7) { - v_vminor = 8396817; + v_vminor = 2105361; if (v_depth == 0) { } else if (0 != (v_expect_after_value & 64)) { - v_vminor = 8396865; + v_vminor = 2105409; } else { - v_vminor = 8396833; + v_vminor = 2105377; } if (v_depth >= 1024) { status = wuffs_base__make_status( @@ -19944,7 +19946,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, (iop_a_src += 1, wuffs_base__make_empty_struct()); if (v_depth <= 1) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(8392738)) + (((uint64_t)(2101282)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__outer__break; @@ -19955,14 +19957,14 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, if (0 == (self->private_data.f_stack[v_stack_byte] & (((uint32_t)(1)) << v_stack_bit))) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(8396834)) + (((uint64_t)(2105378)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_expect = 260; v_expect_after_value = 260; } else { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(8405026)) + (((uint64_t)(2113570)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_expect = 68; @@ -19974,7 +19976,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, 111546413966853); if (v_match == 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(6292481)) + (((uint64_t)(8388612)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(5)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); if (((uint64_t)(io2_a_src - iop_a_src)) < 5) { @@ -19995,7 +19997,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, 435762131972); if (v_match == 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(6293505)) + (((uint64_t)(8388616)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(4)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); if (((uint64_t)(io2_a_src - iop_a_src)) < 4) { @@ -20016,7 +20018,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self, 465676103172); if (v_match == 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(6291969)) + (((uint64_t)(8388610)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(4)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); if (((uint64_t)(io2_a_src - iop_a_src)) < 4) { diff --git a/script/print-json-token-debug-format.c b/script/print-json-token-debug-format.c index 44e2e7dd..494f7dfa 100644 --- a/script/print-json-token-debug-format.c +++ b/script/print-json-token-debug-format.c @@ -128,11 +128,11 @@ read_src() { const char* vbc_names[8] = { "0:Filler..........", // - "1:String..........", // - "2:UnicodeCodePoint", // - "3:Number..........", // - "4:Structure.......", // - "5:Reserved........", // + "1:Structure.......", // + "2:String..........", // + "3:UnicodeCodePoint", // + "4:Literal.........", // + "5:Number..........", // "6:Reserved........", // "7:Reserved........", // }; diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs index beb23a42..f339d6b8 100644 --- a/std/json/decode_json.wuffs +++ b/std/json/decode_json.wuffs @@ -158,7 +158,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { // Emit the leading '"'. args.dst.write_fast_token!( value_major: 0, - value_minor: 0x20_0013, + value_minor: 0x40_0013, link: 0x1, length: 1) args.src.skip32_fast!(actual: 1, worst_case: 1) @@ -177,7 +177,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { if string_length > 0 { args.dst.write_fast_token!( value_major: 0, - value_minor: 0x20_0021, + value_minor: 0x40_0021, link: 0x3, length: string_length) string_length = 0 @@ -197,7 +197,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { if string_length >= 0xFFFB { args.dst.write_fast_token!( value_major: 0, - value_minor: 0x20_0021, + value_minor: 0x40_0021, link: 0x3, length: 0xFFFC) string_length = 0 @@ -210,7 +210,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { if string_length <> 0 { args.dst.write_fast_token!( value_major: 0, - value_minor: 0x20_0021, + value_minor: 0x40_0021, link: 0x3, length: string_length) string_length = 0 @@ -221,7 +221,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { if string_length > 0 { args.dst.write_fast_token!( value_major: 0, - value_minor: 0x20_0021, + value_minor: 0x40_0021, link: 0x3, length: string_length) string_length = 0 @@ -246,7 +246,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { args.src.skip32_fast!(actual: 2, worst_case: 2) args.dst.write_fast_token!( value_major: 0, - value_minor: 0x40_0000 | ((backslash & 0x7F) as base.u32), + value_minor: 0x60_0000 | ((backslash & 0x7F) as base.u32), link: 0x3, length: 2) continue.string_loop @@ -290,7 +290,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { args.src.skip32_fast!(actual: 6, worst_case: 6) args.dst.write_fast_token!( value_major: 0, - value_minor: 0x40_0000 | uni4_value, + value_minor: 0x60_0000 | uni4_value, link: 0x3, length: 6) continue.string_loop @@ -356,7 +356,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { args.src.skip32_fast!(actual: 8, worst_case: 8) args.dst.write_fast_token!( value_major: 0, - value_minor: 0x40_0000 | uni4_high_surrogate | uni4_value, + value_minor: 0x60_0000 | uni4_high_surrogate | uni4_value, link: 0x3, length: 12) continue.string_loop @@ -399,7 +399,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { if string_length >= 0xFFF8 { args.dst.write_fast_token!( value_major: 0, - value_minor: 0x20_0021, + value_minor: 0x40_0021, link: 0x3, length: string_length + 2) string_length = 0 @@ -431,7 +431,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { if string_length >= 0xFFF8 { args.dst.write_fast_token!( value_major: 0, - value_minor: 0x20_0021, + value_minor: 0x40_0021, link: 0x3, length: string_length + 3) string_length = 0 @@ -463,7 +463,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { if string_length >= 0xFFF8 { args.dst.write_fast_token!( value_major: 0, - value_minor: 0x20_0021, + value_minor: 0x40_0021, link: 0x3, length: string_length + 4) string_length = 0 @@ -478,7 +478,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { if string_length > 0 { args.dst.write_fast_token!( value_major: 0, - value_minor: 0x20_0021, + value_minor: 0x40_0021, link: 0x3, length: string_length) string_length = 0 @@ -506,7 +506,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { args.src.skip32_fast!(actual: 1, worst_case: 1) args.dst.write_fast_token!( value_major: 0, - value_minor: 0x20_0013, + value_minor: 0x40_0013, link: 0x2, length: 1) break @@ -557,9 +557,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { { number_length = this.decode_number!(src: args.src) number_status = number_length >> 8 - vminor = 0x60_0004 + vminor = 0xA0_0003 if (number_length & 0x80) <> 0 { - vminor = 0x60_0002 + vminor = 0xA0_0001 } number_length = number_length & 0x7F if number_status == 0 { @@ -597,13 +597,13 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { // -------- END parse numbers. } else if class == 0x05 { // 0x05 is CLASS_OPEN_CURLY_BRACE. - vminor = 0x80_4011 + vminor = 0x20_4011 if depth == 0 { // No-op. } else if 0 <> (expect_after_value & 0x0040) { // 0x0040 is EXPECT_CLOSE_CURLY_BRACE. - vminor = 0x80_4041 + vminor = 0x20_4041 } else { - vminor = 0x80_4021 + vminor = 0x20_4021 } if depth >= 1024 { return "#unsupported recursion depth" @@ -628,7 +628,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { if depth <= 1 { args.dst.write_fast_token!( value_major: 0, - value_minor: 0x80_1042, + value_minor: 0x20_1042, link: 0x0, length: 1) break.outer @@ -639,7 +639,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { if 0 == (this.stack[stack_byte] & ((1 as base.u32) << stack_bit)) { args.dst.write_fast_token!( value_major: 0, - value_minor: 0x80_2042, + value_minor: 0x20_2042, link: 0x0, length: 1) // 0x0104 is (EXPECT_SQUARE_CLOSE_BRACKET | EXPECT_COMMA). @@ -648,7 +648,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { } else { args.dst.write_fast_token!( value_major: 0, - value_minor: 0x80_4042, + value_minor: 0x20_4042, link: 0x0, length: 1) // 0x0044 is (EXPECT_CURLY_CLOSE_BRACE | EXPECT_COMMA). @@ -658,13 +658,13 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { continue.outer } else if class == 0x07 { // 0x07 is CLASS_OPEN_SQUARE_BRACKET. - vminor = 0x80_2011 + vminor = 0x20_2011 if depth == 0 { // No-op. } else if 0 <> (expect_after_value & 0x0040) { // 0x0040 is EXPECT_CLOSE_CURLY_BRACE. - vminor = 0x80_2041 + vminor = 0x20_2041 } else { - vminor = 0x80_2021 + vminor = 0x20_2021 } if depth >= 1024 { return "#unsupported recursion depth" @@ -689,7 +689,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { if depth <= 1 { args.dst.write_fast_token!( value_major: 0, - value_minor: 0x80_1022, + value_minor: 0x20_1022, link: 0x0, length: 1) break.outer @@ -700,7 +700,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { if 0 == (this.stack[stack_byte] & ((1 as base.u32) << stack_bit)) { args.dst.write_fast_token!( value_major: 0, - value_minor: 0x80_2022, + value_minor: 0x20_2022, link: 0x0, length: 1) // 0x0104 is (EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA). @@ -709,7 +709,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { } else { args.dst.write_fast_token!( value_major: 0, - value_minor: 0x80_4022, + value_minor: 0x20_4022, link: 0x0, length: 1) // 0x0044 is (EXPECT_CLOSE_CURLY_BRACE | EXPECT_COMMA). @@ -723,7 +723,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { if match == 0 { args.dst.write_fast_token!( value_major: 0, - value_minor: 0x60_0401, + value_minor: 0x80_0004, link: 0x0, length: 5) if args.src.available() < 5 { @@ -741,7 +741,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { if match == 0 { args.dst.write_fast_token!( value_major: 0, - value_minor: 0x60_0801, + value_minor: 0x80_0008, link: 0x0, length: 4) if args.src.available() < 4 { @@ -759,7 +759,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { if match == 0 { args.dst.write_fast_token!( value_major: 0, - value_minor: 0x60_0201, + value_minor: 0x80_0002, link: 0x0, length: 4) if args.src.available() < 4 { diff --git a/test/data/australian-abc-local-stations.tokens b/test/data/australian-abc-local-stations.tokens index f8b40ee6..0ba8ba72 100644 Binary files a/test/data/australian-abc-local-stations.tokens and b/test/data/australian-abc-local-stations.tokens differ diff --git a/test/data/json-things.unformatted.tokens b/test/data/json-things.unformatted.tokens index 856ea18e..e88cf8c4 100644 Binary files a/test/data/json-things.unformatted.tokens and b/test/data/json-things.unformatted.tokens differ