From 271680333e33f1c7260e5e97b167ba82b2a86daf Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Fri, 24 Jul 2020 13:05:05 +1000 Subject: [PATCH] Have std/cbor parse tags --- example/jsonptr/jsonptr.cc | 50 +++++++++++++++++++++++--- release/c/wuffs-unsupported-snapshot.c | 20 +++++++++++ script/print-json-token-debug-format.c | 4 +-- std/cbor/decode_cbor.wuffs | 28 +++++++++++++++ 4 files changed, 96 insertions(+), 6 deletions(-) diff --git a/example/jsonptr/jsonptr.cc b/example/jsonptr/jsonptr.cc index 567719fe..fdd346a5 100644 --- a/example/jsonptr/jsonptr.cc +++ b/example/jsonptr/jsonptr.cc @@ -321,6 +321,10 @@ wuffs_base__token_buffer g_tok; // g_src.meta.ri). size_t g_curr_token_end_src_index; +// Valid token's VBCs range in 0 ..= 15. Values over that are for tokens from +// outside of the base package, such as the CBOR package. +#define CATEGORY_CBOR_TAG 16 + struct { uint64_t category; uint64_t detail; @@ -1018,6 +1022,10 @@ write_number_as_cbor_u64(uint8_t base, uint64_t u) { const char* // write_cbor_minus_1_minus_x(uint8_t* ptr, size_t len) { + if (g_flags.output_format == file_format::cbor) { + return write_dst(ptr, len); + } + if (len != 9) { return "main: internal error: invalid ETC__MINUS_1_MINUS_X token length"; } @@ -1035,6 +1043,25 @@ write_cbor_minus_1_minus_x(uint8_t* ptr, size_t len) { return write_dst(&buf[0], 1 + n); } +const char* // +write_cbor_tag(uint64_t tag, uint8_t* ptr, size_t len) { + if (g_flags.output_format == file_format::cbor) { + return write_dst(ptr, len); + } + + if (!g_flags.output_cbor_metadata_as_json_comments) { + return nullptr; + } + uint8_t buf[WUFFS_BASE__U64__BYTE_LENGTH__MAX_INCL]; + size_t n = wuffs_base__render_number_u64( + wuffs_base__make_slice_u8(&buf[0], + WUFFS_BASE__U64__BYTE_LENGTH__MAX_INCL), + tag, WUFFS_BASE__RENDER_NUMBER_XXX__DEFAULT_OPTIONS); + TRY(write_dst("/*cbor:tag", 10)); + TRY(write_dst(&buf[0], n)); + return write_dst("*/", 2); +} + const char* // write_number(uint64_t vbd, uint8_t* ptr, size_t len) { if (g_flags.output_format == file_format::json) { @@ -1543,12 +1570,12 @@ handle_token(wuffs_base__token t, bool start_of_token_chain) { int64_t ext = t.value_extension(); if (ext >= 0) { + uint64_t x = (g_token_extension.detail + << WUFFS_BASE__TOKEN__VALUE_EXTENSION__NUM_BITS) | + ((uint64_t)ext); switch (g_token_extension.category) { case WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_SIGNED: case WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_UNSIGNED: - uint64_t x = (g_token_extension.detail - << WUFFS_BASE__TOKEN__VALUE_EXTENSION__NUM_BITS) | - ((uint64_t)ext); TRY(write_inline_integer( x, g_token_extension.category == @@ -1557,13 +1584,28 @@ handle_token(wuffs_base__token t, bool start_of_token_chain) { g_token_extension.category = 0; g_token_extension.detail = 0; goto after_value; + case CATEGORY_CBOR_TAG: + TRY(write_cbor_tag( + x, g_src.data.ptr + g_curr_token_end_src_index - len, len)); + g_token_extension.category = 0; + g_token_extension.detail = 0; + return nullptr; } } if (t.value_major() == WUFFS_CBOR__TOKEN_VALUE_MAJOR) { uint64_t value_minor = t.value_minor(); if (value_minor & WUFFS_CBOR__TOKEN_VALUE_MINOR__TAG) { - // TODO: CBOR tags. + if (t.continued()) { + if (len != 0) { + return "main: internal error: unexpected to-be-extended length"; + } + g_token_extension.category = CATEGORY_CBOR_TAG; + g_token_extension.detail = vbd; + return nullptr; + } + return write_cbor_tag( + vbd, g_src.data.ptr + g_curr_token_end_src_index - len, len); } else if (value_minor & WUFFS_CBOR__TOKEN_VALUE_MINOR__MINUS_1_MINUS_X) { TRY(write_cbor_minus_1_minus_x( g_src.data.ptr + g_curr_token_end_src_index - len, len)); diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index df47df5c..560309be 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -16795,6 +16795,26 @@ wuffs_cbor__decoder__decode_tokens( self->private_data.f_container_num_remaining[v_depth] = v_string_length; v_depth += 1; goto label__outer__continue; + } else if (v_c_major == 6) { + if (v_c_minor >= 28) { + goto label__goto_fail__break; + } + if (v_string_length < 262144) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(787997)) << WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT) | + (((uint64_t)((16777216 | ((uint32_t)(v_string_length))))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(((uint32_t)(WUFFS_CBOR__TOKEN_LENGTHS[v_c_minor])))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + } else { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(787997)) << WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT) | + (((uint64_t)((16777216 | ((uint32_t)((v_string_length >> 46)))))) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__CONTINUED__SHIFT) | + (((uint64_t)(0)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + *iop_a_dst++ = wuffs_base__make_token( + (~(v_string_length & 70368744177663) << WUFFS_BASE__TOKEN__VALUE_EXTENSION__SHIFT) | + (((uint64_t)(((uint32_t)(WUFFS_CBOR__TOKEN_LENGTHS[v_c_minor])))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + } + goto label__outer__continue; } else if (v_c_major == 7) { if (v_c_minor == 31) { if (v_depth <= 0) { diff --git a/script/print-json-token-debug-format.c b/script/print-json-token-debug-format.c index 9ae24f23..03f07403 100644 --- a/script/print-json-token-debug-format.c +++ b/script/print-json-token-debug-format.c @@ -354,8 +354,8 @@ main1(int argc, char** argv) { vmajor_name[3] = g_base38_decode[m3]; } - printf("vmajor=0x%06" PRIX32 ":%s vminor=0x%06" PRIX32 "\n", - vmajor, vmajor_name, vminor); + printf("vmajor=0x%06" PRIX32 ":%s vminor=0x%07" PRIX32 "\n", vmajor, + vmajor_name, vminor); } else if (vmajor == 0) { printf("vbc=%s vbd=0x%06" PRIX32 "\n", g_vbc_names[vbc & 15], vbd); } else { diff --git a/std/cbor/decode_cbor.wuffs b/std/cbor/decode_cbor.wuffs index 40abc7a0..51c95cda 100644 --- a/std/cbor/decode_cbor.wuffs +++ b/std/cbor/decode_cbor.wuffs @@ -531,6 +531,34 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor continue.outer // -------- END Major type 5: a map of pairs of data items. + } else if c_major == 6 { + // -------- BEGIN Major type 6: tags. + if c_minor >= 0x1C { + break.goto_fail + } + // Write one token (18 bits) or two tokens (18 + 46 bits). + if string_length < 0x4_0000 { + args.dst.write_simple_token_fast!( + value_major: TOKEN_VALUE_MAJOR, + value_minor: TOKEN_VALUE_MINOR__TAG | + (string_length as base.u32), + continued: 0, + length: TOKEN_LENGTHS[c_minor] as base.u32) + } else { + args.dst.write_simple_token_fast!( + value_major: TOKEN_VALUE_MAJOR, + value_minor: TOKEN_VALUE_MINOR__TAG | + ((string_length >> base.TOKEN__VALUE_EXTENSION__NUM_BITS) as base.u32), + continued: 1, + length: 0) + args.dst.write_extended_token_fast!( + value_extension: string_length & 0x3FFF_FFFF_FFFF, + continued: 0, + length: TOKEN_LENGTHS[c_minor] as base.u32) + } + continue.outer + // -------- END Major type 6: tags. + } else if c_major == 7 { // -------- BEGIN Major type 7: miscellaneous. if c_minor == 0x1F {