From f8dfc76310854fd5e3101c1661c7c3c94f764206 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Thu, 23 Jul 2020 23:35:44 +1000 Subject: [PATCH] Have std/cbor parse objects --- example/jsonptr/jsonptr.cc | 26 ++++- release/c/wuffs-unsupported-snapshot.c | 97 +++++++++++++--- std/cbor/decode_cbor.wuffs | 150 +++++++++++++++++++++---- 3 files changed, 231 insertions(+), 42 deletions(-) diff --git a/example/jsonptr/jsonptr.cc b/example/jsonptr/jsonptr.cc index c285b27b..567719fe 100644 --- a/example/jsonptr/jsonptr.cc +++ b/example/jsonptr/jsonptr.cc @@ -176,10 +176,14 @@ static const char* g_usage = "input and output formats do not have to match, but conversion between\n" "formats may be lossy.\n" "\n" - "Canonicalized means that e.g. \"abc\\u000A\\tx\\u0177z\" is re-written\n" - "as \"abc\\n\\txŷz\". It does not sort object keys, nor does it reject\n" + "Canonicalized JSON means that e.g. \"abc\\u000A\\tx\\u0177z\" is re-\n" + "written as \"abc\\n\\txŷz\". It does not sort object keys or reject\n" "duplicate keys. Canonicalization does not imply Unicode normalization.\n" "\n" + "CBOR output is non-canonical (in the RFC 7049 Section 3.9 sense), as\n" + "sorting map keys and measuring indefinite-length containers requires\n" + "O(input_length) memory but this program runs in O(1) memory.\n" + "\n" "Formatted means that arrays' and objects' elements are indented, each\n" "on its own line. Configure this with the -c / -compact-output, -s=NUM /\n" "-spaces=NUM (for NUM ranging from 0 to 8) and -t / -tabs flags. Those\n" @@ -207,6 +211,11 @@ static const char* g_usage = "specification but many parsers accept them and they can produce simpler\n" "line-based diffs. This flag is ignored when -compact-output is set.\n" "\n" + "When converting from -i=cbor to -o=json, CBOR permits map keys other\n" + "than (untagged) UTF-8 strings but JSON does not. This program rejects\n" + "such input, as doing otherwise has complicated interactions with the\n" + "-query=STR flag and streaming input.\n" + "\n" "----\n" "\n" "The -q=STR or -query=STR flag gives an optional JSON Pointer query, to\n" @@ -1404,8 +1413,17 @@ handle_token(wuffs_base__token t, bool start_of_token_chain) { } else if (g_ctx == context::in_dict_after_key) { TRY(write_dst(": ", g_flags.compact_output ? 1 : 2)); } else if (g_ctx != context::none) { - if ((g_ctx != context::in_list_after_bracket) && - (g_ctx != context::in_dict_after_brace)) { + if ((g_ctx == context::in_dict_after_brace) || + (g_ctx == context::in_dict_after_value)) { + // Reject dict keys that aren't UTF-8 strings, which could otherwise + // happen with -i=cbor -o=json. + if ((vbc != WUFFS_BASE__TOKEN__VBC__STRING) || + !(vbd & WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8)) { + return "main: cannot convert CBOR non-text-string to JSON map key"; + } + } + if ((g_ctx == context::in_list_after_value) || + (g_ctx == context::in_dict_after_value)) { TRY(write_dst(",", 1)); } if (!g_flags.compact_output) { diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index abb64222..df47df5c 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -5643,7 +5643,6 @@ struct wuffs_cbor__decoder__struct { wuffs_base__vtable null_vtable; bool f_end_of_data; - uint8_t f_container_close_token_length; uint32_t p_decode_tokens[1]; } private_impl; @@ -16422,6 +16421,7 @@ wuffs_cbor__decoder__decode_tokens( uint32_t v_depth = 0; uint32_t v_stack_byte = 0; uint32_t v_stack_bit = 0; + uint32_t v_stack_val = 0; uint32_t v_token_length = 0; uint32_t v_vminor = 0; uint32_t v_vminor_alt = 0; @@ -16760,16 +16760,75 @@ wuffs_cbor__decoder__decode_tokens( self->private_data.f_container_num_remaining[v_depth] = v_string_length; v_depth += 1; goto label__outer__continue; + } else if (v_c_major == 5) { + if (WUFFS_CBOR__TOKEN_LENGTHS[v_c_minor] == 0) { + goto label__goto_fail__break; + } else if (v_depth >= 1024) { + status = wuffs_base__make_status(wuffs_cbor__error__unsupported_recursion_depth); + goto exit; + } + v_vminor = 2113553; + v_vminor_alt = 2101314; + if (v_depth > 0) { + v_stack_byte = ((v_depth - 1) / 16); + v_stack_bit = (((v_depth - 1) & 15) * 2); + if (0 == (self->private_data.f_stack[v_stack_byte] & (((uint32_t)(1)) << v_stack_bit))) { + v_vminor = 2113569; + v_vminor_alt = 2105410; + } else { + v_vminor = 2113601; + v_vminor_alt = 2113602; + } + } + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(v_vminor)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(((uint32_t)(WUFFS_CBOR__TOKEN_LENGTHS[v_c_minor])))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + if (v_c_minor == 0) { + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(v_vminor_alt)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(0)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); + goto label__goto_parsed_a_leaf_value__break; + } + v_stack_byte = (v_depth / 16); + v_stack_bit = ((v_depth & 15) * 2); + self->private_data.f_stack[v_stack_byte] |= (((uint32_t)(3)) << v_stack_bit); + self->private_data.f_container_num_remaining[v_depth] = v_string_length; + v_depth += 1; + goto label__outer__continue; } else if (v_c_major == 7) { if (v_c_minor == 31) { if (v_depth <= 0) { goto label__goto_fail__break; } - if (self->private_data.f_container_num_remaining[(v_depth - 1)] != 0) { + v_depth -= 1; + if (self->private_data.f_container_num_remaining[v_depth] != 0) { goto label__goto_fail__break; } - self->private_data.f_container_num_remaining[(v_depth - 1)] = 1; - self->private_impl.f_container_close_token_length = 1; + v_stack_byte = (v_depth / 16); + v_stack_bit = ((v_depth & 15) * 2); + v_stack_val = (3 & (self->private_data.f_stack[v_stack_byte] >> v_stack_bit)); + if (v_stack_val == 1) { + goto label__goto_fail__break; + } + if (v_stack_val != 3) { + v_vminor_alt = 2097186; + } else { + v_vminor_alt = 2097218; + } + if (v_depth <= 0) { + v_vminor_alt |= 4096; + } else { + v_stack_byte = ((v_depth - 1) / 16); + v_stack_bit = (((v_depth - 1) & 15) * 2); + if (0 == (self->private_data.f_stack[v_stack_byte] & (((uint32_t)(1)) << v_stack_bit))) { + v_vminor_alt |= 8192; + } else { + v_vminor_alt |= 16384; + } + } + *iop_a_dst++ = wuffs_base__make_token( + (((uint64_t)(v_vminor_alt)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | + (((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); goto label__goto_parsed_a_leaf_value__break; } } @@ -16786,6 +16845,12 @@ wuffs_cbor__decoder__decode_tokens( } label__goto_parsed_a_leaf_value__break:; while (v_depth > 0) { + v_stack_byte = ((v_depth - 1) / 16); + v_stack_bit = (((v_depth - 1) & 15) * 2); + self->private_data.f_stack[v_stack_byte] ^= (((uint32_t)(1)) << (v_stack_bit + 1)); + if (1 == (3 & (self->private_data.f_stack[v_stack_byte] >> v_stack_bit))) { + goto label__outer__continue; + } if (self->private_data.f_container_num_remaining[(v_depth - 1)] <= 0) { goto label__outer__continue; } @@ -16799,16 +16864,17 @@ wuffs_cbor__decoder__decode_tokens( WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(8); goto label__2__continue; } - v_vminor_alt = 2097154; - v_stack_byte = ((v_depth - 1) / 16); - v_stack_bit = (((v_depth - 1) & 15) * 2); - if (0 == (self->private_data.f_stack[v_stack_byte] & (((uint32_t)(1)) << v_stack_bit))) { - v_vminor_alt |= 32; - } else { - v_vminor_alt |= 64; - } v_depth -= 1; - if (v_depth > 0) { + v_stack_byte = (v_depth / 16); + v_stack_bit = ((v_depth & 15) * 2); + if (0 == (self->private_data.f_stack[v_stack_byte] & (((uint32_t)(1)) << v_stack_bit))) { + v_vminor_alt = 2097186; + } else { + v_vminor_alt = 2097218; + } + if (v_depth <= 0) { + v_vminor_alt |= 4096; + } else { v_stack_byte = ((v_depth - 1) / 16); v_stack_bit = (((v_depth - 1) & 15) * 2); if (0 == (self->private_data.f_stack[v_stack_byte] & (((uint32_t)(1)) << v_stack_bit))) { @@ -16816,13 +16882,10 @@ wuffs_cbor__decoder__decode_tokens( } else { v_vminor_alt |= 16384; } - } else { - v_vminor_alt |= 4096; } *iop_a_dst++ = wuffs_base__make_token( (((uint64_t)(v_vminor_alt)) << WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT) | - (((uint64_t)(((uint32_t)(self->private_impl.f_container_close_token_length)))) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); - self->private_impl.f_container_close_token_length = 0; + (((uint64_t)(0)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT)); } goto label__outer__break; } diff --git a/std/cbor/decode_cbor.wuffs b/std/cbor/decode_cbor.wuffs index d503f96b..40abc7a0 100644 --- a/std/cbor/decode_cbor.wuffs +++ b/std/cbor/decode_cbor.wuffs @@ -53,8 +53,6 @@ pri const TOKEN_LENGTHS : array[32] base.u8[..= 9] = [ pub struct decoder? implements base.token_decoder( end_of_data : base.bool, - container_close_token_length : base.u8[..= 1], - util : base.utility, )( // stack is conceptually an array of 2-bit integers, implemented as an @@ -89,6 +87,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor var depth : base.u32[..= 1024] var stack_byte : base.u32[..= (1024 / 16) - 1] var stack_bit : base.u32[..= 30] + var stack_val : base.u32[..= 3] var token_length : base.u32[..= 0xFFFF] var vminor : base.u32[..= 0x1FF_FFFF] var vminor_alt : base.u32[..= 0x1FF_FFFF] @@ -471,6 +470,67 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor continue.outer // -------- END Major type 4: an array of data items. + } else if c_major == 5 { + // -------- BEGIN Major type 5: a map of pairs of data items. + if TOKEN_LENGTHS[c_minor] == 0 { + break.goto_fail + } else if depth >= 1024 { + return "#unsupported recursion depth" + } + + vminor = (base.TOKEN__VBC__STRUCTURE << 21) | + base.TOKEN__VBD__STRUCTURE__PUSH | + base.TOKEN__VBD__STRUCTURE__FROM_NONE | + base.TOKEN__VBD__STRUCTURE__TO_DICT + vminor_alt = (base.TOKEN__VBC__STRUCTURE << 21) | + base.TOKEN__VBD__STRUCTURE__POP | + base.TOKEN__VBD__STRUCTURE__FROM_DICT | + base.TOKEN__VBD__STRUCTURE__TO_NONE + if depth > 0 { + stack_byte = (depth - 1) / 16 + stack_bit = ((depth - 1) & 15) * 2 + if 0 == (this.stack[stack_byte] & ((1 as base.u32) << stack_bit)) { + vminor = (base.TOKEN__VBC__STRUCTURE << 21) | + base.TOKEN__VBD__STRUCTURE__PUSH | + base.TOKEN__VBD__STRUCTURE__FROM_LIST | + base.TOKEN__VBD__STRUCTURE__TO_DICT + vminor_alt = (base.TOKEN__VBC__STRUCTURE << 21) | + base.TOKEN__VBD__STRUCTURE__POP | + base.TOKEN__VBD__STRUCTURE__FROM_DICT | + base.TOKEN__VBD__STRUCTURE__TO_LIST + } else { + vminor = (base.TOKEN__VBC__STRUCTURE << 21) | + base.TOKEN__VBD__STRUCTURE__PUSH | + base.TOKEN__VBD__STRUCTURE__FROM_DICT | + base.TOKEN__VBD__STRUCTURE__TO_DICT + vminor_alt = (base.TOKEN__VBC__STRUCTURE << 21) | + base.TOKEN__VBD__STRUCTURE__POP | + base.TOKEN__VBD__STRUCTURE__FROM_DICT | + base.TOKEN__VBD__STRUCTURE__TO_DICT + } + } + args.dst.write_simple_token_fast!( + value_major: 0, + value_minor: vminor, + continued: 0, + length: TOKEN_LENGTHS[c_minor] as base.u32) + if c_minor == 0x00 { + args.dst.write_simple_token_fast!( + value_major: 0, + value_minor: vminor_alt, + continued: 0, + length: 0) + break.goto_parsed_a_leaf_value + } + + stack_byte = depth / 16 + stack_bit = (depth & 15) * 2 + this.stack[stack_byte] |= (3 as base.u32) << stack_bit + this.container_num_remaining[depth] = string_length + depth += 1 + continue.outer + // -------- END Major type 5: a map of pairs of data items. + } else if c_major == 7 { // -------- BEGIN Major type 7: miscellaneous. if c_minor == 0x1F { @@ -479,15 +539,49 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor if depth <= 0 { break.goto_fail } - if this.container_num_remaining[depth - 1] <> 0 { + depth -= 1 + if this.container_num_remaining[depth] <> 0 { break.goto_fail } - // Pretend that the indefinite-length container is effectively - // a definite-length container with one element remaining. - // Having parsed that final phantom element, we'll process the - // explicit CBOR stop code like an implicit end-of-container. - this.container_num_remaining[depth - 1] = 1 - this.container_close_token_length = 1 + + // Check that, if we're in a map, the map contains an even + // number of elements. + stack_byte = depth / 16 + stack_bit = (depth & 15) * 2 + stack_val = 3 & (this.stack[stack_byte] >> stack_bit) + if stack_val == 1 { + break.goto_fail + } + + // Fill in FROM_ETC. + if stack_val <> 3 { + vminor_alt = (base.TOKEN__VBC__STRUCTURE << 21) | + base.TOKEN__VBD__STRUCTURE__POP | + base.TOKEN__VBD__STRUCTURE__FROM_LIST + } else { + vminor_alt = (base.TOKEN__VBC__STRUCTURE << 21) | + base.TOKEN__VBD__STRUCTURE__POP | + base.TOKEN__VBD__STRUCTURE__FROM_DICT + } + + // Fill in TO_ETC. + if depth <= 0 { + vminor_alt |= base.TOKEN__VBD__STRUCTURE__TO_NONE + } else { + stack_byte = (depth - 1) / 16 + stack_bit = ((depth - 1) & 15) * 2 + if 0 == (this.stack[stack_byte] & ((1 as base.u32) << stack_bit)) { + vminor_alt |= base.TOKEN__VBD__STRUCTURE__TO_LIST + } else { + vminor_alt |= base.TOKEN__VBD__STRUCTURE__TO_DICT + } + } + + args.dst.write_simple_token_fast!( + value_major: 0, + value_minor: vminor_alt, + continued: 0, + length: 1) break.goto_parsed_a_leaf_value } // -------- END Major type 7: miscellaneous. @@ -505,6 +599,16 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor // We've just parsed a leaf (non-container) value, or the (explicit or // implicit) close of a container (array or object). while depth > 0 { + // Toggle the key/value bit for object containers. This bit is + // ignored for array containers. + stack_byte = (depth - 1) / 16 + stack_bit = ((depth - 1) & 15) * 2 + this.stack[stack_byte] ^= (1 as base.u32) << (stack_bit + 1) + if 1 == (3 & (this.stack[stack_byte] >> stack_bit)) { + // We just parsed the key of a key-value pair. + continue.outer + } + if this.container_num_remaining[depth - 1] <= 0 { // We're in an indefinite-length container. continue.outer @@ -523,19 +627,25 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor yield? base."$short write" continue } endwhile + depth -= 1 - vminor_alt = (base.TOKEN__VBC__STRUCTURE << 21) | base.TOKEN__VBD__STRUCTURE__POP - - stack_byte = (depth - 1) / 16 - stack_bit = ((depth - 1) & 15) * 2 + // Fill in FROM_ETC. + stack_byte = depth / 16 + stack_bit = (depth & 15) * 2 if 0 == (this.stack[stack_byte] & ((1 as base.u32) << stack_bit)) { - vminor_alt |= base.TOKEN__VBD__STRUCTURE__FROM_LIST + vminor_alt = (base.TOKEN__VBC__STRUCTURE << 21) | + base.TOKEN__VBD__STRUCTURE__POP | + base.TOKEN__VBD__STRUCTURE__FROM_LIST } else { - vminor_alt |= base.TOKEN__VBD__STRUCTURE__FROM_DICT + vminor_alt = (base.TOKEN__VBC__STRUCTURE << 21) | + base.TOKEN__VBD__STRUCTURE__POP | + base.TOKEN__VBD__STRUCTURE__FROM_DICT } - depth -= 1 - if depth > 0 { + // Fill in TO_ETC. + if depth <= 0 { + vminor_alt |= base.TOKEN__VBD__STRUCTURE__TO_NONE + } else { stack_byte = (depth - 1) / 16 stack_bit = ((depth - 1) & 15) * 2 if 0 == (this.stack[stack_byte] & ((1 as base.u32) << stack_bit)) { @@ -543,15 +653,13 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor } else { vminor_alt |= base.TOKEN__VBD__STRUCTURE__TO_DICT } - } else { - vminor_alt |= base.TOKEN__VBD__STRUCTURE__TO_NONE } + args.dst.write_simple_token_fast!( value_major: 0, value_minor: vminor_alt, continued: 0, - length: this.container_close_token_length as base.u32) - this.container_close_token_length = 0 + length: 0) } endwhile // We've parsed the top-level value and we're therefore done overall.