diff --git a/example/jsonptr/jsonptr.cc b/example/jsonptr/jsonptr.cc index 28dbf9be..fb4e230e 100644 --- a/example/jsonptr/jsonptr.cc +++ b/example/jsonptr/jsonptr.cc @@ -312,6 +312,11 @@ wuffs_base__token_buffer g_tok; // g_src.meta.ri). size_t g_curr_token_end_src_index; +struct { + uint64_t category; + uint64_t detail; +} g_token_extension; + uint32_t g_depth; enum class context { @@ -768,6 +773,9 @@ initialize_globals(int argc, char** argv) { g_curr_token_end_src_index = 0; + g_token_extension.category = 0; + g_token_extension.detail = 0; + g_depth = 0; g_ctx = context::none; @@ -1000,48 +1008,22 @@ write_number_as_cbor_u64(uint8_t base, uint64_t u) { } const char* // -write_cbor_number_as_json(uint8_t* ptr, - size_t len, - bool ignore_first_byte, - bool minus_1_minus_x) { - if (ignore_first_byte) { - if (len == 0) { - return "main: internal error: ignore_first_byte with no bytes"; - } - ptr++; - len--; +write_cbor_minus_1_minus_x(uint8_t* ptr, size_t len) { + if (len != 9) { + return "main: internal error: invalid ETC__MINUS_1_MINUS_X token length"; } - uint64_t u; - switch (len) { - case 1: - u = wuffs_base__load_u8__no_bounds_check(ptr); - break; - case 2: - u = wuffs_base__load_u16be__no_bounds_check(ptr); - break; - case 4: - u = wuffs_base__load_u32be__no_bounds_check(ptr); - break; - case 8: - u = wuffs_base__load_u64be__no_bounds_check(ptr); - break; - default: - return "main: internal error: unexpected cbor number byte length"; + uint64_t u = 1 + wuffs_base__load_u64be__no_bounds_check(ptr + 1); + if (u == 0) { + // See the cbor.TOKEN_VALUE_MINOR__MINUS_1_MINUS_X comment re overflow. + return write_dst("-18446744073709551616", 21); } uint8_t buf[1 + WUFFS_BASE__U64__BYTE_LENGTH__MAX_INCL]; uint8_t* b = &buf[0]; - if (minus_1_minus_x) { - u++; - if (u == 0) { - // See the cbor.TOKEN_VALUE_MINOR__MINUS_1_MINUS_X comment re overflow. - return write_dst("-18446744073709551616", 21); - } - *b++ = '-'; - } + *b++ = '-'; size_t n = wuffs_base__render_number_u64( wuffs_base__make_slice_u8(b, WUFFS_BASE__U64__BYTE_LENGTH__MAX_INCL), u, WUFFS_BASE__RENDER_NUMBER_XXX__DEFAULT_OPTIONS); - return write_dst(&buf[0], n + (minus_1_minus_x ? 1 : 0)); + return write_dst(&buf[0], 1 + n); } const char* // @@ -1049,19 +1031,9 @@ write_number(uint64_t vbd, uint8_t* ptr, size_t len) { if (g_flags.output_format == file_format::json) { if (g_flags.input_format == file_format::json) { return write_dst(ptr, len); - } else if ((vbd & - WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_UNSIGNED) && - (vbd & - WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_BIG_ENDIAN)) { - return write_cbor_number_as_json( - ptr, len, - vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_IGNORE_FIRST_BYTE, - false); } // From here on, (g_flags.output_format == file_format::cbor). - } else if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_BIG_ENDIAN) { - return write_dst(ptr, len); } else if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_TEXT) { // First try to parse (ptr, len) as an integer. Something like // "1180591620717411303424" is a valid number (in the JSON sense) but will @@ -1536,6 +1508,11 @@ handle_token(wuffs_base__token t, bool start_of_token_chain) { uint64_t x = x_is_signed ? ((uint64_t)(t.value_base_detail__sign_extended())) : vbd; + if (t.continued()) { + g_token_extension.category = vbc; + g_token_extension.detail = x; + return nullptr; + } TRY(write_inline_integer( x, x_is_signed, g_src.data.ptr + g_curr_token_end_src_index - len, len)); @@ -1543,14 +1520,32 @@ handle_token(wuffs_base__token t, bool start_of_token_chain) { } } + int64_t ext = t.value_extension(); + if (ext >= 0) { + switch (g_token_extension.category) { + case WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_SIGNED: + case WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_UNSIGNED: + uint64_t x = (g_token_extension.detail + << WUFFS_BASE__TOKEN__VALUE_EXTENSION__NUM_BITS) | + ((uint64_t)ext); + TRY(write_inline_integer( + x, + g_token_extension.category == + WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_SIGNED, + g_src.data.ptr + g_curr_token_end_src_index - len, len)); + g_token_extension.category = 0; + g_token_extension.detail = 0; + goto after_value; + } + } + if (t.value_major() == WUFFS_CBOR__TOKEN_VALUE_MAJOR) { uint64_t value_minor = t.value_minor(); if (value_minor & WUFFS_CBOR__TOKEN_VALUE_MINOR__TAG) { // TODO: CBOR tags. } else if (value_minor & WUFFS_CBOR__TOKEN_VALUE_MINOR__MINUS_1_MINUS_X) { - TRY(write_cbor_number_as_json( - g_src.data.ptr + g_curr_token_end_src_index - len, len, true, - true)); + TRY(write_cbor_minus_1_minus_x( + g_src.data.ptr + g_curr_token_end_src_index - len, len)); goto after_value; } } diff --git a/internal/cgen/base/token-public.h b/internal/cgen/base/token-public.h index 24a10a7f..0f74e2b9 100644 --- a/internal/cgen/base/token-public.h +++ b/internal/cgen/base/token-public.h @@ -56,6 +56,8 @@ wuffs_base__make_token(uint64_t repr) { #define WUFFS_BASE__TOKEN__CONTINUED__SHIFT 16 #define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0 +#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__NUM_BITS 46 + // -------- #define WUFFS_BASE__TOKEN__VBC__FILLER 0 diff --git a/internal/cgen/builtin.go b/internal/cgen/builtin.go index 56f3d658..6eeccb8a 100644 --- a/internal/cgen/builtin.go +++ b/internal/cgen/builtin.go @@ -356,29 +356,39 @@ func (g *gen) writeBuiltinIOWriter(b *buffer, recv *a.Expr, method t.ID, args [] } func (g *gen) writeBuiltinTokenWriter(b *buffer, recv *a.Expr, method t.ID, args []*a.Node, depth uint32) error { - if method == t.IDWriteSimpleTokenFast { - b.printf("*iop_a_dst++ = wuffs_base__make_token(\n(((uint64_t)(") + switch method { + case t.IDWriteSimpleTokenFast, t.IDWriteExtendedTokenFast: + b.printf("*iop_a_dst++ = wuffs_base__make_token(\n") - if cv := args[0].AsArg().Value().ConstValue(); (cv == nil) || (cv.Sign() != 0) { + if method == t.IDWriteSimpleTokenFast { + b.writes("(((uint64_t)(") + if cv := args[0].AsArg().Value().ConstValue(); (cv == nil) || (cv.Sign() != 0) { + if err := g.writeExpr(b, args[0].AsArg().Value(), depth); err != nil { + return err + } + b.writes(")) << WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT) |\n(((uint64_t)(") + } + + if err := g.writeExpr(b, args[1].AsArg().Value(), depth); err != nil { + return err + } + b.writes(")) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |\n(((uint64_t)(") + } else { + b.writes("(~") if err := g.writeExpr(b, args[0].AsArg().Value(), depth); err != nil { return err } - b.writes(")) << WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT) |\n(((uint64_t)(") + b.writes(" << WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT) |\n(((uint64_t)(") } - if err := g.writeExpr(b, args[1].AsArg().Value(), depth); err != nil { - return err - } - b.writes(")) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) |\n(((uint64_t)(") - - if cv := args[2].AsArg().Value().ConstValue(); (cv == nil) || (cv.Sign() != 0) { - if err := g.writeExpr(b, args[2].AsArg().Value(), depth); err != nil { + if cv := args[len(args)-2].AsArg().Value().ConstValue(); (cv == nil) || (cv.Sign() != 0) { + if err := g.writeExpr(b, args[len(args)-2].AsArg().Value(), depth); err != nil { return err } b.writes(")) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) |\n(((uint64_t)(") } - if err := g.writeExpr(b, args[3].AsArg().Value(), depth); err != nil { + if err := g.writeExpr(b, args[len(args)-1].AsArg().Value(), depth); err != nil { return err } b.writes(")) << WUFFS_BASE__TOKEN__LENGTH__SHIFT))") diff --git a/internal/cgen/data/data.go b/internal/cgen/data/data.go index 457337d0..29f84404 100644 --- a/internal/cgen/data/data.go +++ b/internal/cgen/data/data.go @@ -567,7 +567,7 @@ const BaseTokenPrivateH = "" + const BaseTokenPublicH = "" + "// ---------------- Tokens\n\n// wuffs_base__token is an element of a byte stream's tokenization.\n//\n// See https://github.com/google/wuffs/blob/master/doc/note/tokens.md\ntypedef struct {\n uint64_t repr;\n\n#ifdef __cplusplus\n inline int64_t value() const;\n inline int64_t value_extension() const;\n inline int64_t value_major() const;\n inline int64_t value_base_category() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_detail() const;\n inline int64_t value_base_detail__sign_extended() const;\n inline bool continued() const;\n inline uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\n" + "" + - "// --------\n\n#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 17\n#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 17\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 17\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 38\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 17\n#define WUFFS_BASE__TOKEN__CONTINUED__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n" + + "// --------\n\n#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 17\n#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT 17\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 17\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 38\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 17\n#define WUFFS_BASE__TOKEN__CONTINUED__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__NUM_BITS 46\n\n" + "" + "// --------\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1\n#define WUFFS_BASE__TOKEN__VBC__STRING 2\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 3\n#define WUFFS_BASE__TOKEN__VBC__LITERAL 4\n#define WUFFS_BASE__TOKEN__VBC__NUMBER 5\n#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_SIGNED 6\n#define WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_UNSIGNED 7\n\n" + "" + diff --git a/lang/builtin/builtin.go b/lang/builtin/builtin.go index 2ccc0c5e..edb7baed 100644 --- a/lang/builtin/builtin.go +++ b/lang/builtin/builtin.go @@ -75,6 +75,10 @@ var Consts = [...]struct { // ---- + {t.IDU32, "46", "TOKEN__VALUE_EXTENSION__NUM_BITS"}, + + // ---- + {t.IDU32, "0", "TOKEN__VBC__FILLER"}, {t.IDU32, "1", "TOKEN__VBC__STRUCTURE"}, {t.IDU32, "2", "TOKEN__VBC__STRING"}, @@ -464,6 +468,9 @@ var Funcs = []string{ "token_writer.write_simple_token_fast!(" + "value_major: u32[..= 0x1F_FFFF], value_minor: u32[..= 0x1FF_FFFF]," + "continued: u32[..= 0x1], length: u32[..= 0xFFFF])", + "token_writer.write_extended_token_fast!(" + + "value_extension: u64[..= 0x3FFF_FFFF_FFFF]," + + "continued: u32[..= 0x1], length: u32[..= 0xFFFF])", "token_writer.available() u64", diff --git a/lang/check/bounds.go b/lang/check/bounds.go index 4a1e8a4f..b0ddc149 100644 --- a/lang/check/bounds.go +++ b/lang/check/bounds.go @@ -1264,7 +1264,8 @@ var ioMethodAdvances = [...]struct { t.IDWriteU64BEFast - t.IDPeekU8: {eight, true}, t.IDWriteU64LEFast - t.IDPeekU8: {eight, true}, - t.IDWriteSimpleTokenFast - t.IDPeekU8: {one, true}, + t.IDWriteSimpleTokenFast - t.IDPeekU8: {one, true}, + t.IDWriteExtendedTokenFast - t.IDPeekU8: {one, true}, } func makeConstValueExpr(tm *t.Map, cv *big.Int) (*a.Expr, error) { diff --git a/lang/token/list.go b/lang/token/list.go index 4e876267..2f48fc54 100644 --- a/lang/token/list.go +++ b/lang/token/list.go @@ -629,8 +629,8 @@ const ( // -------- - // TODO: IDWriteExtendedTokenFast - IDWriteSimpleTokenFast = ID(0x1F1) + IDWriteSimpleTokenFast = ID(0x1F1) + IDWriteExtendedTokenFast = ID(0x1F2) // -------- 0x200 block. @@ -994,7 +994,8 @@ var builtInsByID = [nBuiltInIDs]string{ // -------- - IDWriteSimpleTokenFast: "write_simple_token_fast", + IDWriteSimpleTokenFast: "write_simple_token_fast", + IDWriteExtendedTokenFast: "write_extended_token_fast", // -------- 0x200 block. diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index fcbb5bd6..c9571338 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -2126,6 +2126,8 @@ wuffs_base__make_token(uint64_t repr) { #define WUFFS_BASE__TOKEN__CONTINUED__SHIFT 16 #define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0 +#define WUFFS_BASE__TOKEN__VALUE_EXTENSION__NUM_BITS 46 + // -------- #define WUFFS_BASE__TOKEN__VBC__FILLER 0 @@ -16254,6 +16256,14 @@ const char wuffs_cbor__error__internal_error_inconsistent_token_length[] = "#cbo // ---------------- Private Consts +static const uint8_t +WUFFS_CBOR__TOKEN_LENGTHS[32]WUFFS_BASE__POTENTIALLY_UNUSED = { + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 2, 3, 5, 9, 1, 1, 1, 1, +}; + // ---------------- Private Initializer Prototypes // ---------------- Private Function Prototypes @@ -16451,7 +16461,7 @@ wuffs_cbor__decoder__decode_tokens( while (true) { while (true) { while (true) { - if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) { + if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 1) { status = wuffs_base__make_status(wuffs_base__suspension__short_write); WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(1); goto label__outer__continue; @@ -16535,28 +16545,42 @@ wuffs_cbor__decoder__decode_tokens( label__goto_have_string_length__break:; } if (v_c_major == 0) { - if (v_c_minor < 24) { + if (v_c_minor < 26) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)((14680064 | ((uint32_t)(v_c_minor))))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + (((uint64_t)((14680064 | ((uint32_t)((v_string_length & 65535)))))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(((uint32_t)(WUFFS_CBOR__TOKEN_LENGTHS[v_c_minor])))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__goto_parsed_a_leaf_value__break; } else if (v_c_minor < 28) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(10490116)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)((1 + (((uint32_t)(1)) << (v_c_minor - 24))))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + (((uint64_t)((14680064 | ((uint32_t)((v_string_length >> 46)))))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | + (((uint64_t)(0)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + *iop_a_dst++ = wuffs_base__make_token( + (~(v_string_length & 70368744177663) << WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT) | + (((uint64_t)(((uint32_t)(WUFFS_CBOR__TOKEN_LENGTHS[v_c_minor])))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__goto_parsed_a_leaf_value__break; } } else if (v_c_major == 1) { - if (v_c_minor < 24) { + if (v_c_minor < 26) { *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)((12582912 | (2097151 - ((uint32_t)(v_c_minor)))))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + (((uint64_t)((12582912 | (2097151 - ((uint32_t)((v_string_length & 65535))))))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(((uint32_t)(WUFFS_CBOR__TOKEN_LENGTHS[v_c_minor])))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__goto_parsed_a_leaf_value__break; } else if (v_c_minor < 28) { - *iop_a_dst++ = wuffs_base__make_token( - (((uint64_t)(787997)) << WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT) | - (((uint64_t)(2)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)((1 + (((uint32_t)(1)) << (v_c_minor - 24))))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + if (v_string_length < 9223372036854775808u) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)((12582912 | (2097151 - ((uint32_t)((v_string_length >> 46))))))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | + (((uint64_t)(0)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + *iop_a_dst++ = wuffs_base__make_token( + (~((18446744073709551615u - v_string_length) & 70368744177663) << WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT) | + (((uint64_t)(((uint32_t)(WUFFS_CBOR__TOKEN_LENGTHS[v_c_minor])))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + } else { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(787997)) << WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT) | + (((uint64_t)(2)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(9)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + } goto label__goto_parsed_a_leaf_value__break; } } else if (v_c_major == 2) { diff --git a/std/cbor/decode_cbor.wuffs b/std/cbor/decode_cbor.wuffs index 1766c742..e4e179c0 100644 --- a/std/cbor/decode_cbor.wuffs +++ b/std/cbor/decode_cbor.wuffs @@ -26,13 +26,20 @@ pub const TOKEN_VALUE_MAJOR : base.u32 = 0x0C_061D // tag is ((first << 46) | ext). pub const TOKEN_VALUE_MINOR__TAG : base.u32 = 0x01 -// TOKEN_VALUE_MINOR__MINUS_1_MINUS_X means that the token's final -// (token.length - 1) bytes (which must be one of 1, 2, 4 or 8 bytes) contains -// a big-endian unsigned integer x. The token's overall value is the negative -// number (-1 - x). Note that this value can underflow an int64_t, and its -// absolute value (+1 + x) can overflow a uint64_t. +// TOKEN_VALUE_MINOR__MINUS_1_MINUS_X means that the 9-byte length token holds +// the negative integer (-1 - x), where x is the big-endian unsigned integer in +// the token's final 8 bytes. The most significant bit of x is guaranteed to be +// set, so that (-1 - x) will always underflow an int64_t and its absolute +// value (+1 + x) might also overflow a uint64_t. pub const TOKEN_VALUE_MINOR__MINUS_1_MINUS_X : base.u32 = 0x02 +pri const TOKEN_LENGTHS : array[32] base.u8[..= 9] = [ + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 2, 3, 5, 9, 1, 1, 1, 1, +] + pub struct decoder? implements base.token_decoder( end_of_data : base.bool, @@ -56,8 +63,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor var value_minor : base.u32[..= 0x1FF_FFFF] var continued : base.u32[..= 1] var c : base.u8 - var c_major : base.u8 - var c_minor : base.u8 + var c_major : base.u8[..= 0x07] + var c_minor : base.u8[..= 0x1F] // indefinite_string_major_type is 2 or 3 when we are in an // indefinite-length byte string or text string. It is 0 otherwise. @@ -70,7 +77,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor while.outer true { while.goto_parsed_a_leaf_value true {{ while.goto_fail true {{ - if args.dst.available() <= 0 { + if args.dst.available() <= 1 { yield? base."$short write" continue.outer } @@ -111,7 +118,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor string_length = c_minor as base.u64 } else { while.goto_have_string_length true, - inv args.dst.available() > 0, + inv args.dst.available() > 1, {{ if c_minor == 0x18 { if args.src.available() >= 1 { @@ -158,43 +165,58 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor if c_major == 0 { // -------- BEGIN Major type 0: an unsigned integer. - if c_minor < 0x18 { + if c_minor < 0x1A { args.dst.write_simple_token_fast!( value_major: 0, value_minor: (base.TOKEN__VBC__INLINE_INTEGER_UNSIGNED << 21) | - (c_minor as base.u32), + ((string_length & 0xFFFF) as base.u32), continued: 0, - length: 1) + length: TOKEN_LENGTHS[c_minor] as base.u32) break.goto_parsed_a_leaf_value } else if c_minor < 0x1C { args.dst.write_simple_token_fast!( value_major: 0, - value_minor: (base.TOKEN__VBC__NUMBER << 21) | - base.TOKEN__VBD__NUMBER__CONTENT_INTEGER_UNSIGNED | - base.TOKEN__VBD__NUMBER__FORMAT_BINARY_BIG_ENDIAN | - base.TOKEN__VBD__NUMBER__FORMAT_IGNORE_FIRST_BYTE, + value_minor: (base.TOKEN__VBC__INLINE_INTEGER_UNSIGNED << 21) | + ((string_length >> base.TOKEN__VALUE_EXTENSION__NUM_BITS) as base.u32), + continued: 1, + length: 0) + args.dst.write_extended_token_fast!( + value_extension: string_length & 0x3FFF_FFFF_FFFF, continued: 0, - length: 1 + ((1 as base.u32) << (c_minor - 0x18))) + length: TOKEN_LENGTHS[c_minor] as base.u32) break.goto_parsed_a_leaf_value } // -------- END Major type 0: an unsigned integer. } else if c_major == 1 { // -------- END Major type 1: a negative integer. - if c_minor < 0x18 { + if c_minor < 0x1A { args.dst.write_simple_token_fast!( value_major: 0, value_minor: (base.TOKEN__VBC__INLINE_INTEGER_SIGNED << 21) | - (0x1F_FFFF - (c_minor as base.u32)), + (0x1F_FFFF - ((string_length & 0xFFFF) as base.u32)), continued: 0, - length: 1) + length: TOKEN_LENGTHS[c_minor] as base.u32) break.goto_parsed_a_leaf_value } else if c_minor < 0x1C { - args.dst.write_simple_token_fast!( - value_major: TOKEN_VALUE_MAJOR, - value_minor: TOKEN_VALUE_MINOR__MINUS_1_MINUS_X, - continued: 0, - length: 1 + ((1 as base.u32) << (c_minor - 0x18))) + if string_length < 0x8000_0000_0000_0000 { + args.dst.write_simple_token_fast!( + value_major: 0, + value_minor: (base.TOKEN__VBC__INLINE_INTEGER_SIGNED << 21) | + (0x1F_FFFF - ((string_length >> base.TOKEN__VALUE_EXTENSION__NUM_BITS) as base.u32)), + continued: 1, + length: 0) + args.dst.write_extended_token_fast!( + value_extension: (0xFFFF_FFFF_FFFF_FFFF - string_length) & 0x3FFF_FFFF_FFFF, + continued: 0, + length: TOKEN_LENGTHS[c_minor] as base.u32) + } else { + args.dst.write_simple_token_fast!( + value_major: TOKEN_VALUE_MAJOR, + value_minor: TOKEN_VALUE_MINOR__MINUS_1_MINUS_X, + continued: 0, + length: 9) + } break.goto_parsed_a_leaf_value } // -------- END Major type 1: a negative integer.