diff --git a/internal/cgen/base/token-public.h b/internal/cgen/base/token-public.h index 8eeb80dc..87c84f5d 100644 --- a/internal/cgen/base/token-public.h +++ b/internal/cgen/base/token-public.h @@ -71,8 +71,9 @@ wuffs_base__make_token(uint64_t repr) { // -------- -#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00001 +#define WUFFS_BASE__TOKEN__VBD__FILLER__PUNCTUATION 0x00001 #define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00002 +#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00004 // -------- diff --git a/internal/cgen/data/data.go b/internal/cgen/data/data.go index 4bc5dc63..c7eb412c 100644 --- a/internal/cgen/data/data.go +++ b/internal/cgen/data/data.go @@ -315,7 +315,7 @@ const BaseTokenPublicH = "" + "" + "// --------\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1\n#define WUFFS_BASE__TOKEN__VBC__STRING 2\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 3\n#define WUFFS_BASE__TOKEN__VBC__LITERAL 4\n#define WUFFS_BASE__TOKEN__VBC__NUMBER 5\n#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_SIGNED 6\n#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_UNSIGNED 7\n\n" + "" + - "// --------\n\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00001\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00002\n\n" + + "// --------\n\n#define WUFFS_BASE__TOKEN__VBD__FILLER__PUNCTUATION 0x00001\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00002\n#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00004\n\n" + "" + "// --------\n\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__PUSH 0x00001\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__POP 0x00002\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_NONE 0x00010\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST 0x00020\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_DICT 0x00040\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_NONE 0x01000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST 0x02000\n#define WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_DICT 0x04000\n\n" + "" + diff --git a/lang/builtin/builtin.go b/lang/builtin/builtin.go index ed29f3ea..a497daf4 100644 --- a/lang/builtin/builtin.go +++ b/lang/builtin/builtin.go @@ -90,8 +90,9 @@ var Consts = [...]struct { // ---- - {t.IDU32, "0x00001", "TOKEN__VBD__FILLER__COMMENT_LINE"}, + {t.IDU32, "0x00001", "TOKEN__VBD__FILLER__PUNCTUATION"}, {t.IDU32, "0x00002", "TOKEN__VBD__FILLER__COMMENT_BLOCK"}, + {t.IDU32, "0x00004", "TOKEN__VBD__FILLER__COMMENT_LINE"}, // ---- diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 04f69321..908cebd0 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -2243,8 +2243,9 @@ wuffs_base__make_token(uint64_t repr) { // -------- -#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00001 +#define WUFFS_BASE__TOKEN__VBD__FILLER__PUNCTUATION 0x00001 #define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_BLOCK 0x00002 +#define WUFFS_BASE__TOKEN__VBD__FILLER__COMMENT_LINE 0x00004 // -------- @@ -26618,7 +26619,7 @@ wuffs_json__decoder__decode_tokens( } else if (v_class == 2) { (iop_a_src += 1, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); if (0 == (v_expect & (((uint32_t)(1)) << 8))) { if (self->private_impl.f_quirks[13]) { @@ -26637,7 +26638,7 @@ wuffs_json__decoder__decode_tokens( } else if (v_class == 3) { (iop_a_src += 1, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_expect = 7858; goto label__outer__continue; @@ -27372,7 +27373,7 @@ wuffs_json__decoder__decode_comment( if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) { if (v_length > 0) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(4)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)(v_length)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); } @@ -27389,7 +27390,7 @@ wuffs_json__decoder__decode_comment( if (v_c == 10) { (iop_a_src += 1, wuffs_base__make_empty_struct()); *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(4)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)((v_length + 1))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); status = wuffs_base__make_status(NULL); goto ok; @@ -27397,7 +27398,7 @@ wuffs_json__decoder__decode_comment( (iop_a_src += 1, wuffs_base__make_empty_struct()); if (v_length >= 65533) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(1)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(4)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | (((uint64_t)((v_length + 1))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); v_length = 0; diff --git a/std/json/decode_json.wuffs b/std/json/decode_json.wuffs index cd601c55..8c3bd7e7 100644 --- a/std/json/decode_json.wuffs +++ b/std/json/decode_json.wuffs @@ -899,10 +899,11 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor } else if class == CLASS_COMMA { args.src.skip_u32_fast!(actual: 1, worst_case: 1) - // The ',' is filler. + // The ',' is punctuation (filler). args.dst.write_simple_token_fast!( value_major: 0, - value_minor: 0, + value_minor: (base.TOKEN__VBC__FILLER << 21) | + base.TOKEN__VBD__FILLER__PUNCTUATION, continued: 0, length: 1) // What's valid after a comma depends on whether or not we're in an @@ -924,10 +925,11 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor } else if class == CLASS_COLON { args.src.skip_u32_fast!(actual: 1, worst_case: 1) - // The ':' is filler. + // The ':' is punctuation (filler). args.dst.write_simple_token_fast!( value_major: 0, - value_minor: 0, + value_minor: (base.TOKEN__VBC__FILLER << 21) | + base.TOKEN__VBD__FILLER__PUNCTUATION, continued: 0, length: 1) expect = EXPECT_VALUE diff --git a/test/data/australian-abc-local-stations.tokens b/test/data/australian-abc-local-stations.tokens index 8eba91c0..d17a0c3b 100644 Binary files a/test/data/australian-abc-local-stations.tokens and b/test/data/australian-abc-local-stations.tokens differ diff --git a/test/data/json-quirks.tokens b/test/data/json-quirks.tokens index 3a36b054..d7b08ab4 100644 Binary files a/test/data/json-quirks.tokens and b/test/data/json-quirks.tokens differ diff --git a/test/data/json-things.unformatted.tokens b/test/data/json-things.unformatted.tokens index 1fd7fb86..447662de 100644 Binary files a/test/data/json-things.unformatted.tokens and b/test/data/json-things.unformatted.tokens differ