From 0291a47dc80abd390012871b85351029e5f699c6 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Thu, 13 Aug 2020 22:40:10 +1000 Subject: [PATCH] Remove example/jsonptr's ability to speak CBOR Originally, the example/jsonptr program read JSON from stdin and wrote JSON to stdout. Over the course of numerous recent commits, it gained the ability to read either CBOR or JSON and write either CBOR or JSON. (CBOR is, very roughly, a binary file format similar to the JSON text file format). While a useful feature, the extra code hurt JSON-to-JSON performance, the original motivating purpose of that program. This commit reverts that feature, re-gaining original performance. (The example/cbor-to-json and example/json-to-cbor programs were recently added to convert between CBOR and JSON, covering that feature loss). On a mid-range x86_64 laptop, processing the 181 MiB citylots.json file from github.com/zemirco/sf-city-lots-json: $ time gen/bin/example-jsonptr < citylots.json > /dev/null Before this commit: real 0m5.095s After: real 0m3.583s Ratio: 1.42x $ sloccount example/jsonptr/jsonptr.cc | grep ^cpp Before this commit: cpp: 1508 (100.00%) After: cpp: 869 (100.00%) Ratio: 1.74x This commit brings jsonptr.cc approximately to its state just before 168f60a7 "Add example/jsonptr -output-format=cbor flag" was committed, although it keeps the flags added since then for allowing comments and inf/NaN numbers. --- example/jsonptr/jsonptr.cc | 1005 +++++------------------------------- 1 file changed, 133 insertions(+), 872 deletions(-) diff --git a/example/jsonptr/jsonptr.cc b/example/jsonptr/jsonptr.cc index c4ff7df2..eab2cb28 100644 --- a/example/jsonptr/jsonptr.cc +++ b/example/jsonptr/jsonptr.cc @@ -16,8 +16,8 @@ /* jsonptr is a JSON formatter (pretty-printer) that supports the JSON Pointer -(RFC 6901) query syntax. It reads CBOR or UTF-8 JSON from stdin and writes CBOR -or canonicalized, formatted UTF-8 JSON to stdout. +(RFC 6901) query syntax. It reads UTF-8 JSON from stdin and writes +canonicalized, formatted UTF-8 JSON to stdout. See the "const char* g_usage" string below for details. @@ -27,23 +27,23 @@ JSON Pointer (and this program's implementation) is one of many JSON query languages and JSON tools, such as jq, jql and JMESPath. This one is relatively simple and fewer-featured compared to those others. -One benefit of simplicity is that this program's CBOR, JSON and JSON Pointer +One benefit of simplicity is that this program's JSON and JSON Pointer implementations do not dynamically allocate or free memory (yet it does not require that the entire input fits in memory at once). They are therefore trivially protected against certain bug classes: memory leaks, double-frees and use-after-frees. -The CBOR and JSON implementations are also written in the Wuffs programming -language (and then transpiled to C/C++), which is memory-safe (e.g. array -indexing is bounds-checked) but also prevents integer arithmetic overflows. +The core JSON implementation is also written in the Wuffs programming language +(and then transpiled to C/C++), which is memory-safe (e.g. array indexing is +bounds-checked) but also guards against integer arithmetic overflows. For defense in depth, on Linux, this program also self-imposes a SECCOMP_MODE_STRICT sandbox before reading (or otherwise processing) its input or writing its output. Under this sandbox, the only permitted system calls are read, write, exit and sigreturn. -All together, this program aims to safely handle untrusted CBOR or JSON files -without fear of security bugs such as remote code execution. +All together, this program aims to safely handle untrusted JSON files without +fear of security bugs such as remote code execution. ---- @@ -120,7 +120,6 @@ for a C++ compiler $CXX, such as clang++ or g++. // code simply isn't compiled. #define WUFFS_CONFIG__MODULES #define WUFFS_CONFIG__MODULE__BASE -#define WUFFS_CONFIG__MODULE__CBOR #define WUFFS_CONFIG__MODULE__JSON // If building this program in an environment that doesn't easily accommodate @@ -152,18 +151,14 @@ static const char* g_usage = "Flags:\n" " -c -compact-output\n" " -d=NUM -max-output-depth=NUM\n" - " -i=FMT -input-format={json,cbor}\n" - " -o=FMT -output-format={json,cbor}\n" " -q=STR -query=STR\n" " -s=NUM -spaces=NUM\n" " -t -tabs\n" " -fail-if-unsandboxed\n" - " -input-allow-json-comments\n" - " -input-allow-json-extra-comma\n" - " -input-allow-json-inf-nan-numbers\n" - " -output-cbor-metadata-as-json-comments\n" - " -output-json-extra-comma\n" - " -output-json-inf-nan-numbers\n" + " -input-allow-comments\n" + " -input-allow-extra-comma\n" + " -input-allow-inf-nan-numbers\n" + " -output-extra-comma\n" " -strict-json-pointer-syntax\n" "\n" "The input.json filename is optional. If absent, it reads from stdin.\n" @@ -171,57 +166,32 @@ static const char* g_usage = "----\n" "\n" "jsonptr is a JSON formatter (pretty-printer) that supports the JSON\n" - "Pointer (RFC 6901) query syntax. It reads CBOR or UTF-8 JSON from stdin\n" - "and writes CBOR or canonicalized, formatted UTF-8 JSON to stdout. The\n" - "input and output formats do not have to match, but conversion between\n" - "formats may be lossy.\n" + "Pointer (RFC 6901) query syntax. It reads UTF-8 JSON from stdin and\n" + "writes canonicalized, formatted UTF-8 JSON to stdout.\n" "\n" - "Canonicalized JSON means that e.g. \"abc\\u000A\\tx\\u0177z\" is re-\n" - "written as \"abc\\n\\txŷz\". It does not sort object keys or reject\n" + "Canonicalized means that e.g. \"abc\\u000A\\tx\\u0177z\" is re-written\n" + "as \"abc\\n\\txŷz\". It does not sort object keys, nor does it reject\n" "duplicate keys. Canonicalization does not imply Unicode normalization.\n" "\n" - "CBOR output is non-canonical (in the RFC 7049 Section 3.9 sense), as\n" - "sorting map keys and measuring indefinite-length containers requires\n" - "O(input_length) memory but this program runs in O(1) memory.\n" - "\n" "Formatted means that arrays' and objects' elements are indented, each\n" "on its own line. Configure this with the -c / -compact-output, -s=NUM /\n" - "-spaces=NUM (for NUM ranging from 0 to 8) and -t / -tabs flags. Those\n" - "flags only apply to JSON (not CBOR) output.\n" + "-spaces=NUM (for NUM ranging from 0 to 8) and -t / -tabs flags.\n" "\n" - "The -input-format and -output-format flags select between reading and\n" - "writing JSON (the default, a textual format) or CBOR (a binary format).\n" + "The -input-allow-comments flag allows \"/*slash-star*/\" and\n" + "\"//slash-slash\" C-style comments within JSON input. Such comments are\n" + "stripped from the output.\n" "\n" - "The -input-allow-json-comments flag allows \"/*slash-star*/\" and\n" - "\"//slash-slash\" C-style comments within JSON input.\n" + "The -input-allow-extra-comma flag allows input like \"[1,2,]\", with a\n" + "comma after the final element of a JSON list or dictionary.\n" "\n" - "The -input-allow-json-extra-comma flag allows input like \"[1,2,]\",\n" - "with a comma after the final element of a JSON list or dictionary.\n" + "The -input-allow-inf-nan-numbers flag allows non-finite floating point\n" + "numbers (infinities and not-a-numbers) within JSON input.\n" "\n" - "The -input-allow-json-inf-nan-numbers flag allows non-finite floating\n" - "point numbers (infinities and not-a-numbers) within JSON input.\n" - "\n" - "The -output-cbor-metadata-as-json-comments writes CBOR tags and other\n" - "metadata as /*comments*/, when -i=json and -o=cbor are also set. Such\n" - "comments are non-compliant with the JSON specification but many parsers\n" - "accept them.\n" - "\n" - "The -output-json-extra-comma flag writes extra commas, regardless of\n" - "whether the input had it. Such commas are non-compliant with the JSON\n" - "specification but many parsers accept them and they can produce simpler\n" - "line-based diffs. This flag is ignored when -compact-output is set.\n" - "\n" - "The -output-json-inf-nan-numbers flag writes Inf and NaN instead of a\n" - "substitute null value, when converting from -i=cbor to -o=json. Such\n" - "values are non-compliant with the JSON specification but many parsers\n" - "accept them.\n" - "\n" - "CBOR is more permissive about map keys but JSON only allows strings.\n" - "When converting from -i=cbor to -o=json, this program rejects keys other\n" - "than text strings and non-negative integers (CBOR major types 3 and 0).\n" - "Integer keys like 123 quoted to be string keys like \"123\". Being even\n" - "more permissive would have complicated interactions with the -query=STR\n" - "flag and streaming input, so this program just rejects other keys.\n" + "The -output-extra-comma flag writes output like \"[1,2,]\", with a comma\n" + "after the final element of a JSON list or dictionary. Such commas are\n" + "non-compliant with the JSON specification but many parsers accept them\n" + "and they can produce simpler line-based diffs. This flag is ignored when\n" + "-compact-output is set.\n" "\n" "----\n" "\n" @@ -238,18 +208,17 @@ static const char* g_usage = "child (the value in a key-value pair) of the root whose key is the empty\n" "string. Similarly, \"/xyz\" and \"/xyz/\" are two different nodes.\n" "\n" - "If the query found a valid JSON|CBOR value, this program will return a\n" - "zero exit code even if the rest of the input isn't valid. If the query\n" + "If the query found a valid JSON value, this program will return a zero\n" + "exit code even if the rest of the input isn't valid JSON. If the query\n" "did not find a value, or found an invalid one, this program returns a\n" "non-zero exit code, but may still print partial output to stdout.\n" "\n" - "The JSON and CBOR specifications (https://json.org/ or RFC 8259; RFC\n" - "7049) permit implementations to allow duplicate keys, as this one does.\n" - "This JSON Pointer implementation is also greedy, following the first\n" - "match for each fragment without back-tracking. For example, the\n" - "\"/foo/bar\" query will fail if the root object has multiple \"foo\"\n" - "children but the first one doesn't have a \"bar\" child, even if later\n" - "ones do.\n" + "The JSON specification (https://json.org/) permits implementations that\n" + "allow duplicate keys, as this one does. This JSON Pointer implementation\n" + "is also greedy, following the first match for each fragment without\n" + "back-tracking. For example, the \"/foo/bar\" query will fail if the root\n" + "object has multiple \"foo\" children but the first one doesn't have a\n" + "\"bar\" child, even if later ones do.\n" "\n" "The -strict-json-pointer-syntax flag restricts the -query=STR string to\n" "exactly RFC 6901, with only two escape sequences: \"~0\" and \"~1\" for\n" @@ -261,15 +230,15 @@ static const char* g_usage = "----\n" "\n" "The -d=NUM or -max-output-depth=NUM flag gives the maximum (inclusive)\n" - "output depth. JSON|CBOR containers ([] arrays and {} objects) can hold\n" - "other containers. When this flag is set, containers at depth NUM are\n" - "replaced with \"[…]\" or \"{…}\". A bare -d or -max-output-depth is\n" - "equivalent to -d=1. The flag's absence means an unlimited output depth.\n" + "output depth. JSON containers ([] arrays and {} objects) can hold other\n" + "containers. When this flag is set, containers at depth NUM are replaced\n" + "with \"[…]\" or \"{…}\". A bare -d or -max-output-depth is equivalent to\n" + "-d=1. The flag's absence is equivalent to an unlimited output depth.\n" "\n" "The -max-output-depth flag only affects the program's output. It doesn't\n" - "affect whether or not the input is considered valid JSON|CBOR. The\n" - "format specifications permit implementations to set their own maximum\n" - "input depth. This JSON|CBOR implementation sets it to 1024.\n" + "affect whether or not the input is considered valid JSON. The JSON\n" + "specification permits implementations to set their own maximum input\n" + "depth. This JSON implementation sets it to 1024.\n" "\n" "Depth is measured in terms of nested containers. It is unaffected by the\n" "number of spaces or tabs used to indent.\n" @@ -329,17 +298,6 @@ wuffs_base__token_buffer g_tok; // g_src.meta.ri). size_t g_curr_token_end_src_index; -// Valid token's VBCs range in 0 ..= 15. Values over that are for tokens from -// outside of the base package, such as the CBOR package. -#define CATEGORY_CBOR_TAG 16 - -struct { - uint64_t category; - uint64_t detail; -} g_token_extension; - -bool g_previous_token_was_cbor_tag; - uint32_t g_depth; enum class context { @@ -360,32 +318,7 @@ in_dict_before_key() { uint32_t g_suppress_write_dst; bool g_wrote_to_dst; -wuffs_cbor__decoder g_cbor_decoder; -wuffs_json__decoder g_json_decoder; -wuffs_base__token_decoder* g_dec; - -// g_spool_array is a 4 KiB buffer. -// -// For -o=cbor, strings up to SPOOL_ARRAY_SIZE long are written as a single -// definite-length string. Longer strings are written as an indefinite-length -// string containing multiple definite-length chunks, each of length up to -// SPOOL_ARRAY_SIZE. See RFC 7049 section 2.2.2 "Indefinite-Length Byte Strings -// and Text Strings". Byte strings and text strings are spooled prior to this -// chunking, so that the output is determinate even when the input is streamed. -// -// For -o=json, CBOR byte strings are spooled prior to base64url encoding, -// which map multiples of 3 source bytes to 4 destination bytes. -// -// If raising SPOOL_ARRAY_SIZE above 0xFFFF then you will also have to update -// flush_cbor_output_string. -#define SPOOL_ARRAY_SIZE 4096 -uint8_t g_spool_array[SPOOL_ARRAY_SIZE]; - -uint32_t g_cbor_output_string_length; -bool g_cbor_output_string_is_multiple_chunks; -bool g_cbor_output_string_is_utf_8; - -uint32_t g_json_output_byte_string_length; +wuffs_json__decoder g_dec; // ---- @@ -467,7 +400,6 @@ class Query { uint32_t m_depth; wuffs_base__result_u64 m_array_index; - uint64_t m_array_index_remaining; public: void reset(char* query_c_string) { @@ -477,7 +409,6 @@ class Query { m_depth = 0; m_array_index.status.repr = "#main: not an array index query fragment"; m_array_index.value = 0; - m_array_index_remaining = 0; } void restart_fragment(bool enable) { m_frag_j = enable ? m_frag_i : nullptr; } @@ -488,10 +419,10 @@ class Query { // zero. If valid but non-zero, it decrements it and returns false. bool tick() { if (m_array_index.status.is_ok()) { - if (m_array_index_remaining == 0) { + if (m_array_index.value == 0) { return true; } - m_array_index_remaining--; + m_array_index.value--; } return false; } @@ -523,7 +454,6 @@ class Query { m_array_index = wuffs_base__parse_number_u64( wuffs_base__make_slice_u8(i, k - i), WUFFS_BASE__PARSE_NUMBER_XXX__DEFAULT_OPTIONS); - m_array_index_remaining = m_array_index.value; } return true; } @@ -532,13 +462,6 @@ class Query { bool matched_fragment() { return m_frag_j && (m_frag_j == m_frag_k); } - void restart_and_match_unsigned_number(bool enable, uint64_t u) { - m_frag_j = - (enable && (m_array_index.status.is_ok()) && (m_array_index.value == u)) - ? m_frag_k - : nullptr; - } - void incremental_match_slice(uint8_t* ptr, size_t len) { if (!m_frag_j) { return; @@ -649,26 +572,17 @@ class Query { // ---- -enum class file_format { - json, - cbor, -}; - struct { int remaining_argc; char** remaining_argv; bool compact_output; bool fail_if_unsandboxed; - file_format input_format; - bool input_allow_json_comments; - bool input_allow_json_extra_comma; - bool input_allow_json_inf_nan_numbers; + bool input_allow_comments; + bool input_allow_extra_comma; + bool input_allow_inf_nan_numbers; uint32_t max_output_depth; - file_format output_format; - bool output_cbor_metadata_as_json_comments; - bool output_json_extra_comma; - bool output_json_inf_nan_numbers; + bool output_extra_comma; char* query_c_string; size_t spaces; bool strict_json_pointer_syntax; @@ -724,44 +638,20 @@ parse_flags(int argc, char** argv) { g_flags.fail_if_unsandboxed = true; continue; } - if (!strcmp(arg, "i=cbor") || !strcmp(arg, "input-format=cbor")) { - g_flags.input_format = file_format::cbor; + if (!strcmp(arg, "input-allow-comments")) { + g_flags.input_allow_comments = true; continue; } - if (!strcmp(arg, "i=json") || !strcmp(arg, "input-format=json")) { - g_flags.input_format = file_format::json; + if (!strcmp(arg, "input-allow-extra-comma")) { + g_flags.input_allow_extra_comma = true; continue; } - if (!strcmp(arg, "input-allow-json-comments")) { - g_flags.input_allow_json_comments = true; + if (!strcmp(arg, "input-allow-inf-nan-numbers")) { + g_flags.input_allow_inf_nan_numbers = true; continue; } - if (!strcmp(arg, "input-allow-json-extra-comma")) { - g_flags.input_allow_json_extra_comma = true; - continue; - } - if (!strcmp(arg, "input-allow-json-inf-nan-numbers")) { - g_flags.input_allow_json_inf_nan_numbers = true; - continue; - } - if (!strcmp(arg, "o=cbor") || !strcmp(arg, "output-format=cbor")) { - g_flags.output_format = file_format::cbor; - continue; - } - if (!strcmp(arg, "o=json") || !strcmp(arg, "output-format=json")) { - g_flags.output_format = file_format::json; - continue; - } - if (!strcmp(arg, "output-cbor-metadata-as-json-comments")) { - g_flags.output_cbor_metadata_as_json_comments = true; - continue; - } - if (!strcmp(arg, "output-json-extra-comma")) { - g_flags.output_json_extra_comma = true; - continue; - } - if (!strcmp(arg, "output-json-inf-nan-numbers")) { - g_flags.output_json_inf_nan_numbers = true; + if (!strcmp(arg, "output-extra-comma")) { + g_flags.output_extra_comma = true; continue; } if (!strncmp(arg, "q=", 2) || !strncmp(arg, "query=", 6)) { @@ -818,11 +708,6 @@ initialize_globals(int argc, char** argv) { g_curr_token_end_src_index = 0; - g_token_extension.category = 0; - g_token_extension.detail = 0; - - g_previous_token_was_cbor_tag = false; - g_depth = 0; g_ctx = context::none; @@ -844,34 +729,25 @@ initialize_globals(int argc, char** argv) { g_suppress_write_dst = g_query.next_fragment() ? 1 : 0; g_wrote_to_dst = false; - if (g_flags.input_format == file_format::json) { - TRY(g_json_decoder - .initialize(sizeof__wuffs_json__decoder(), WUFFS_VERSION, 0) - .message()); - g_dec = g_json_decoder.upcast_as__wuffs_base__token_decoder(); - } else { - TRY(g_cbor_decoder - .initialize(sizeof__wuffs_cbor__decoder(), WUFFS_VERSION, 0) - .message()); - g_dec = g_cbor_decoder.upcast_as__wuffs_base__token_decoder(); - } + TRY(g_dec.initialize(sizeof__wuffs_json__decoder(), WUFFS_VERSION, 0) + .message()); - if (g_flags.input_allow_json_comments) { - g_dec->set_quirk_enabled(WUFFS_JSON__QUIRK_ALLOW_COMMENT_BLOCK, true); - g_dec->set_quirk_enabled(WUFFS_JSON__QUIRK_ALLOW_COMMENT_LINE, true); + if (g_flags.input_allow_comments) { + g_dec.set_quirk_enabled(WUFFS_JSON__QUIRK_ALLOW_COMMENT_BLOCK, true); + g_dec.set_quirk_enabled(WUFFS_JSON__QUIRK_ALLOW_COMMENT_LINE, true); } - if (g_flags.input_allow_json_extra_comma) { - g_dec->set_quirk_enabled(WUFFS_JSON__QUIRK_ALLOW_EXTRA_COMMA, true); + if (g_flags.input_allow_extra_comma) { + g_dec.set_quirk_enabled(WUFFS_JSON__QUIRK_ALLOW_EXTRA_COMMA, true); } - if (g_flags.input_allow_json_inf_nan_numbers) { - g_dec->set_quirk_enabled(WUFFS_JSON__QUIRK_ALLOW_INF_NAN_NUMBERS, true); + if (g_flags.input_allow_inf_nan_numbers) { + g_dec.set_quirk_enabled(WUFFS_JSON__QUIRK_ALLOW_INF_NAN_NUMBERS, true); } // Consume an optional whitespace trailer. This isn't part of the JSON spec, // but it works better with line oriented Unix tools (such as "echo 123 | // jsonptr" where it's "echo", not "echo -n") or hand-edited JSON files which // can accidentally contain trailing whitespace. - g_dec->set_quirk_enabled(WUFFS_JSON__QUIRK_ALLOW_TRAILING_NEW_LINE, true); + g_dec.set_quirk_enabled(WUFFS_JSON__QUIRK_ALLOW_TRAILING_NEW_LINE, true); return nullptr; } @@ -957,300 +833,6 @@ write_dst(const void* s, size_t n) { // ---- -const char* // -write_literal(uint64_t vbd) { - const char* ptr = nullptr; - size_t len = 0; - if (vbd & WUFFS_BASE__TOKEN__VBD__LITERAL__UNDEFINED) { - if (g_flags.output_format == file_format::json) { - // JSON's closest approximation to "undefined" is "null". - if (g_flags.output_cbor_metadata_as_json_comments) { - ptr = "/*cbor:undefined*/null"; - len = 22; - } else { - ptr = "null"; - len = 4; - } - } else { - ptr = "\xF7"; - len = 1; - } - } else if (vbd & WUFFS_BASE__TOKEN__VBD__LITERAL__NULL) { - if (g_flags.output_format == file_format::json) { - ptr = "null"; - len = 4; - } else { - ptr = "\xF6"; - len = 1; - } - } else if (vbd & WUFFS_BASE__TOKEN__VBD__LITERAL__FALSE) { - if (g_flags.output_format == file_format::json) { - ptr = "false"; - len = 5; - } else { - ptr = "\xF4"; - len = 1; - } - } else if (vbd & WUFFS_BASE__TOKEN__VBD__LITERAL__TRUE) { - if (g_flags.output_format == file_format::json) { - ptr = "true"; - len = 4; - } else { - ptr = "\xF5"; - len = 1; - } - } else { - return "main: internal error: unexpected write_literal argument"; - } - return write_dst(ptr, len); -} - -// ---- - -const char* // -write_number_as_cbor_f64(double f) { - uint8_t buf[9]; - wuffs_base__lossy_value_u16 lv16 = - wuffs_base__ieee_754_bit_representation__from_f64_to_u16_truncate(f); - if (!lv16.lossy) { - buf[0] = 0xF9; - wuffs_base__store_u16be__no_bounds_check(&buf[1], lv16.value); - return write_dst(&buf[0], 3); - } - wuffs_base__lossy_value_u32 lv32 = - wuffs_base__ieee_754_bit_representation__from_f64_to_u32_truncate(f); - if (!lv32.lossy) { - buf[0] = 0xFA; - wuffs_base__store_u32be__no_bounds_check(&buf[1], lv32.value); - return write_dst(&buf[0], 5); - } - buf[0] = 0xFB; - wuffs_base__store_u64be__no_bounds_check( - &buf[1], wuffs_base__ieee_754_bit_representation__from_f64_to_u64(f)); - return write_dst(&buf[0], 9); -} - -const char* // -write_number_as_cbor_u64(uint8_t base, uint64_t u) { - uint8_t buf[9]; - if (u < 0x18) { - buf[0] = base | ((uint8_t)u); - return write_dst(&buf[0], 1); - } else if ((u >> 8) == 0) { - buf[0] = base | 0x18; - buf[1] = ((uint8_t)u); - return write_dst(&buf[0], 2); - } else if ((u >> 16) == 0) { - buf[0] = base | 0x19; - wuffs_base__store_u16be__no_bounds_check(&buf[1], ((uint16_t)u)); - return write_dst(&buf[0], 3); - } else if ((u >> 32) == 0) { - buf[0] = base | 0x1A; - wuffs_base__store_u32be__no_bounds_check(&buf[1], ((uint32_t)u)); - return write_dst(&buf[0], 5); - } - buf[0] = base | 0x1B; - wuffs_base__store_u64be__no_bounds_check(&buf[1], u); - return write_dst(&buf[0], 9); -} - -const char* // -write_number_as_json_f64(wuffs_base__slice_u8 s) { - double f; - switch (s.len) { - case 3: - f = wuffs_base__ieee_754_bit_representation__from_u16_to_f64( - wuffs_base__load_u16be__no_bounds_check(s.ptr + 1)); - break; - case 5: - f = wuffs_base__ieee_754_bit_representation__from_u32_to_f64( - wuffs_base__load_u32be__no_bounds_check(s.ptr + 1)); - break; - case 9: - f = wuffs_base__ieee_754_bit_representation__from_u64_to_f64( - wuffs_base__load_u64be__no_bounds_check(s.ptr + 1)); - break; - default: - return "main: internal error: unexpected write_number_as_json_f64 len"; - } - uint8_t buf[512]; - const uint32_t precision = 0; - size_t n = wuffs_base__render_number_f64( - wuffs_base__make_slice_u8(&buf[0], sizeof buf), f, precision, - WUFFS_BASE__RENDER_NUMBER_FXX__JUST_ENOUGH_PRECISION); - - if (!g_flags.output_json_inf_nan_numbers) { - // JSON numbers don't include Infinities or NaNs. For such numbers, their - // IEEE 754 bit representation's 11 exponent bits are all on. - uint64_t u = wuffs_base__ieee_754_bit_representation__from_f64_to_u64(f); - if (((u >> 52) & 0x7FF) == 0x7FF) { - if (g_flags.output_cbor_metadata_as_json_comments) { - TRY(write_dst("/*cbor:", 7)); - TRY(write_dst(&buf[0], n)); - TRY(write_dst("*/", 2)); - } - return write_dst("null", 4); - } - } - - return write_dst(&buf[0], n); -} - -const char* // -write_cbor_minus_1_minus_x(wuffs_base__slice_u8 s) { - if (g_flags.output_format == file_format::cbor) { - return write_dst(s.ptr, s.len); - } - - if (s.len != 9) { - return "main: internal error: invalid ETC__MINUS_1_MINUS_X token length"; - } - uint64_t u = 1 + wuffs_base__load_u64be__no_bounds_check(s.ptr + 1); - if (u == 0) { - // See the cbor.TOKEN_VALUE_MINOR__MINUS_1_MINUS_X comment re overflow. - return write_dst("-18446744073709551616", 21); - } - uint8_t buf[1 + WUFFS_BASE__U64__BYTE_LENGTH__MAX_INCL]; - uint8_t* b = &buf[0]; - *b++ = '-'; - size_t n = wuffs_base__render_number_u64( - wuffs_base__make_slice_u8(b, WUFFS_BASE__U64__BYTE_LENGTH__MAX_INCL), u, - WUFFS_BASE__RENDER_NUMBER_XXX__DEFAULT_OPTIONS); - return write_dst(&buf[0], 1 + n); -} - -const char* // -write_cbor_simple_value(uint64_t tag, wuffs_base__slice_u8 s) { - if (g_flags.output_format == file_format::cbor) { - return write_dst(s.ptr, s.len); - } - - if (!g_flags.output_cbor_metadata_as_json_comments) { - return write_dst("null", 4); - } - uint8_t buf[WUFFS_BASE__U64__BYTE_LENGTH__MAX_INCL]; - size_t n = wuffs_base__render_number_u64( - wuffs_base__make_slice_u8(&buf[0], - WUFFS_BASE__U64__BYTE_LENGTH__MAX_INCL), - tag, WUFFS_BASE__RENDER_NUMBER_XXX__DEFAULT_OPTIONS); - TRY(write_dst("/*cbor:simple", 13)); - TRY(write_dst(&buf[0], n)); - return write_dst("*/null", 6); -} - -const char* // -write_cbor_tag(uint64_t tag, wuffs_base__slice_u8 s) { - if (g_flags.output_format == file_format::cbor) { - return write_dst(s.ptr, s.len); - } - - if (!g_flags.output_cbor_metadata_as_json_comments) { - return nullptr; - } - uint8_t buf[WUFFS_BASE__U64__BYTE_LENGTH__MAX_INCL]; - size_t n = wuffs_base__render_number_u64( - wuffs_base__make_slice_u8(&buf[0], - WUFFS_BASE__U64__BYTE_LENGTH__MAX_INCL), - tag, WUFFS_BASE__RENDER_NUMBER_XXX__DEFAULT_OPTIONS); - TRY(write_dst("/*cbor:tag", 10)); - TRY(write_dst(&buf[0], n)); - return write_dst("*/", 2); -} - -const char* // -write_number(uint64_t vbd, wuffs_base__slice_u8 s) { - const uint64_t cfp_fbbe_fifb = - WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT | - WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_BINARY_BIG_ENDIAN | - WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_IGNORE_FIRST_BYTE; - - if (g_flags.output_format == file_format::json) { - if (g_flags.input_format == file_format::json) { - return write_dst(s.ptr, s.len); - } else if ((vbd & cfp_fbbe_fifb) == cfp_fbbe_fifb) { - return write_number_as_json_f64(s); - } - - // From here on, (g_flags.output_format == file_format::cbor). - } else if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__FORMAT_TEXT) { - // First try to parse s as an integer. Something like - // "1180591620717411303424" is a valid number (in the JSON sense) but will - // overflow int64_t or uint64_t, so fall back to parsing it as a float64. - if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_INTEGER_SIGNED) { - if ((s.len > 0) && (s.ptr[0] == '-')) { - wuffs_base__result_i64 ri = wuffs_base__parse_number_i64( - s, WUFFS_BASE__PARSE_NUMBER_XXX__DEFAULT_OPTIONS); - if (ri.status.is_ok()) { - return write_number_as_cbor_u64(0x20, ~ri.value); - } - } else { - wuffs_base__result_u64 ru = wuffs_base__parse_number_u64( - s, WUFFS_BASE__PARSE_NUMBER_XXX__DEFAULT_OPTIONS); - if (ru.status.is_ok()) { - return write_number_as_cbor_u64(0x00, ru.value); - } - } - } - - if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT) { - wuffs_base__result_f64 rf = wuffs_base__parse_number_f64( - s, WUFFS_BASE__PARSE_NUMBER_XXX__DEFAULT_OPTIONS); - if (rf.status.is_ok()) { - return write_number_as_cbor_f64(rf.value); - } - } - } else if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_NEG_INF) { - return write_dst("\xF9\xFC\x00", 3); - } else if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_POS_INF) { - return write_dst("\xF9\x7C\x00", 3); - } else if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_NEG_NAN) { - return write_dst("\xF9\xFF\xFF", 3); - } else if (vbd & WUFFS_BASE__TOKEN__VBD__NUMBER__CONTENT_POS_NAN) { - return write_dst("\xF9\x7F\xFF", 3); - } else if ((vbd & cfp_fbbe_fifb) == cfp_fbbe_fifb) { - return write_dst(s.ptr, s.len); - } - -fail: - return "main: internal error: unexpected write_number argument"; -} - -const char* // -write_inline_integer(uint64_t x, bool x_is_signed, wuffs_base__slice_u8 s) { - bool is_key = in_dict_before_key(); - g_query.restart_and_match_unsigned_number( - is_key && g_query.is_at(g_depth) && !x_is_signed, x); - - if (g_flags.output_format == file_format::cbor) { - return write_dst(s.ptr, s.len); - } - - if (is_key) { - TRY(write_dst("\"", 1)); - } - - // Adding the two ETC__BYTE_LENGTH__ETC constants is overkill, but it's - // simpler (for producing a constant-expression array size) than taking the - // maximum of the two. - uint8_t buf[WUFFS_BASE__I64__BYTE_LENGTH__MAX_INCL + - WUFFS_BASE__U64__BYTE_LENGTH__MAX_INCL]; - wuffs_base__slice_u8 dst = wuffs_base__make_slice_u8(&buf[0], sizeof buf); - size_t n = - x_is_signed - ? wuffs_base__render_number_i64( - dst, (int64_t)x, WUFFS_BASE__RENDER_NUMBER_XXX__DEFAULT_OPTIONS) - : wuffs_base__render_number_u64( - dst, x, WUFFS_BASE__RENDER_NUMBER_XXX__DEFAULT_OPTIONS); - TRY(write_dst(&buf[0], n)); - - if (is_key) { - TRY(write_dst("\"", 1)); - } - return nullptr; -} - -// ---- - uint8_t // hex_digit(uint8_t nibble) { nibble &= 0x0F; @@ -1260,188 +842,38 @@ hex_digit(uint8_t nibble) { return ('A' - 10) + nibble; } -const char* // -flush_cbor_output_string() { - uint8_t prefix[3]; - prefix[0] = g_cbor_output_string_is_utf_8 ? 0x60 : 0x40; - if (g_cbor_output_string_length < 0x18) { - prefix[0] |= g_cbor_output_string_length; - TRY(write_dst(&prefix[0], 1)); - } else if (g_cbor_output_string_length <= 0xFF) { - prefix[0] |= 0x18; - prefix[1] = g_cbor_output_string_length; - TRY(write_dst(&prefix[0], 2)); - } else if (g_cbor_output_string_length <= 0xFFFF) { - prefix[0] |= 0x19; - prefix[1] = g_cbor_output_string_length >> 8; - prefix[2] = g_cbor_output_string_length; - TRY(write_dst(&prefix[0], 3)); - } else { - return "main: internal error: CBOR string output is too long"; - } - - size_t n = g_cbor_output_string_length; - g_cbor_output_string_length = 0; - return write_dst(&g_spool_array[0], n); -} - -const char* // -write_cbor_output_string(wuffs_base__slice_u8 s, bool finish) { - // Check that g_spool_array can hold any UTF-8 code point. - if (SPOOL_ARRAY_SIZE < 4) { - return "main: internal error: SPOOL_ARRAY_SIZE is too short"; - } - - uint8_t* ptr = s.ptr; - size_t len = s.len; - while (len > 0) { - size_t available = SPOOL_ARRAY_SIZE - g_cbor_output_string_length; - if (available >= len) { - memcpy(&g_spool_array[g_cbor_output_string_length], ptr, len); - g_cbor_output_string_length += len; - ptr += len; - len = 0; - break; - - } else if (available > 0) { - if (!g_cbor_output_string_is_multiple_chunks) { - g_cbor_output_string_is_multiple_chunks = true; - TRY(write_dst(g_cbor_output_string_is_utf_8 ? "\x7F" : "\x5F", 1)); - } - - if (g_cbor_output_string_is_utf_8) { - // Walk the end backwards to a UTF-8 boundary, so that each chunk of - // the multi-chunk string is also valid UTF-8. - while (available > 0) { - wuffs_base__utf_8__next__output o = - wuffs_base__utf_8__next_from_end(ptr, available); - if ((o.code_point != WUFFS_BASE__UNICODE_REPLACEMENT_CHARACTER) || - (o.byte_length != 1)) { - break; - } - available--; - } - } - - memcpy(&g_spool_array[g_cbor_output_string_length], ptr, available); - g_cbor_output_string_length += available; - ptr += available; - len -= available; - } - - TRY(flush_cbor_output_string()); - } - - if (finish) { - TRY(flush_cbor_output_string()); - if (g_cbor_output_string_is_multiple_chunks) { - TRY(write_dst("\xFF", 1)); - } - } - return nullptr; -} - -const char* // -flush_json_output_byte_string(bool finish) { - uint8_t* ptr = &g_spool_array[0]; - size_t len = g_json_output_byte_string_length; - while (len > 0) { - wuffs_base__transform__output o = wuffs_base__base_64__encode( - g_dst.writer_slice(), wuffs_base__make_slice_u8(ptr, len), finish, - WUFFS_BASE__BASE_64__URL_ALPHABET); - g_dst.meta.wi += o.num_dst; - ptr += o.num_src; - len -= o.num_src; - if (o.status.repr == nullptr) { - if (len != 0) { - return "main: internal error: inconsistent spool length"; - } - g_json_output_byte_string_length = 0; - break; - } else if (o.status.repr == wuffs_base__suspension__short_read) { - memmove(&g_spool_array[0], ptr, len); - g_json_output_byte_string_length = len; - break; - } else if (o.status.repr != wuffs_base__suspension__short_write) { - return o.status.message(); - } - TRY(flush_dst()); - } - return nullptr; -} - -const char* // -write_json_output_byte_string(wuffs_base__slice_u8 s, bool finish) { - // This function and flush_json_output_byte_string doesn't call write_dst. - // Instead, they call wuffs_base__base_64__encode to write directly to g_dst. - // It is therefore responsible for checking g_suppress_write_dst. - if (g_suppress_write_dst > 0) { - return nullptr; - } - - uint8_t* ptr = s.ptr; - size_t len = s.len; - while (len > 0) { - size_t available = SPOOL_ARRAY_SIZE - g_json_output_byte_string_length; - if (available >= len) { - memcpy(&g_spool_array[g_json_output_byte_string_length], ptr, len); - g_json_output_byte_string_length += len; - ptr += len; - len = 0; - break; - - } else if (available > 0) { - memcpy(&g_spool_array[g_json_output_byte_string_length], ptr, available); - g_json_output_byte_string_length += available; - ptr += available; - len -= available; - } - - TRY(flush_json_output_byte_string(false)); - } - - if (finish) { - TRY(flush_json_output_byte_string(true)); - } - return nullptr; -} - -// ---- - const char* // handle_unicode_code_point(uint32_t ucp) { - if (g_flags.output_format == file_format::json) { - if (ucp < 0x0020) { - switch (ucp) { - case '\b': - return write_dst("\\b", 2); - case '\f': - return write_dst("\\f", 2); - case '\n': - return write_dst("\\n", 2); - case '\r': - return write_dst("\\r", 2); - case '\t': - return write_dst("\\t", 2); - } - - // Other bytes less than 0x0020 are valid UTF-8 but not valid in a - // JSON string. They need to remain escaped. - uint8_t esc6[6]; - esc6[0] = '\\'; - esc6[1] = 'u'; - esc6[2] = '0'; - esc6[3] = '0'; - esc6[4] = hex_digit(ucp >> 4); - esc6[5] = hex_digit(ucp >> 0); - return write_dst(&esc6[0], 6); - - } else if (ucp == '\"') { - return write_dst("\\\"", 2); - - } else if (ucp == '\\') { - return write_dst("\\\\", 2); + if (ucp < 0x0020) { + switch (ucp) { + case '\b': + return write_dst("\\b", 2); + case '\f': + return write_dst("\\f", 2); + case '\n': + return write_dst("\\n", 2); + case '\r': + return write_dst("\\r", 2); + case '\t': + return write_dst("\\t", 2); } + + // Other bytes less than 0x0020 are valid UTF-8 but not valid in a + // JSON string. They need to remain escaped. + uint8_t esc6[6]; + esc6[0] = '\\'; + esc6[1] = 'u'; + esc6[2] = '0'; + esc6[3] = '0'; + esc6[4] = hex_digit(ucp >> 4); + esc6[5] = hex_digit(ucp >> 0); + return write_dst(&esc6[0], 6); + + } else if (ucp == '\"') { + return write_dst("\\\"", 2); + + } else if (ucp == '\\') { + return write_dst("\\\\", 2); } uint8_t u[WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL]; @@ -1452,91 +884,7 @@ handle_unicode_code_point(uint32_t ucp) { if (n == 0) { return "main: internal error: unexpected Unicode code point"; } - - if (g_flags.output_format == file_format::json) { - return write_dst(&u[0], n); - } - return write_cbor_output_string(wuffs_base__make_slice_u8(&u[0], n), false); -} - -const char* // -write_json_output_text_string(wuffs_base__slice_u8 s) { - uint8_t* ptr = s.ptr; - size_t len = s.len; -restart: - while (true) { - size_t i; - for (i = 0; i < len; i++) { - uint8_t c = ptr[i]; - if ((c == '"') || (c == '\\') || (c < 0x20)) { - TRY(write_dst(ptr, i)); - TRY(handle_unicode_code_point(c)); - ptr += i + 1; - len -= i + 1; - goto restart; - } - } - TRY(write_dst(ptr, len)); - break; - } - return nullptr; -} - -const char* // -handle_string(uint64_t vbd, - wuffs_base__slice_u8 s, - bool start_of_token_chain, - bool continued) { - if (start_of_token_chain) { - if (g_flags.output_format == file_format::json) { - if (g_flags.output_cbor_metadata_as_json_comments && - !(vbd & WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8)) { - TRY(write_dst("/*cbor:base64url*/\"", 19)); - g_json_output_byte_string_length = 0; - } else { - TRY(write_dst("\"", 1)); - } - } else { - g_cbor_output_string_length = 0; - g_cbor_output_string_is_multiple_chunks = false; - g_cbor_output_string_is_utf_8 = - vbd & WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8; - } - g_query.restart_fragment(in_dict_before_key() && g_query.is_at(g_depth)); - } - - if (vbd & WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP) { - // No-op. - } else if (vbd & WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY) { - if (g_flags.output_format == file_format::json) { - if (g_flags.input_format == file_format::json) { - TRY(write_dst(s.ptr, s.len)); - } else if (vbd & WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8) { - TRY(write_json_output_text_string(s)); - } else { - TRY(write_json_output_byte_string(s, false)); - } - } else { - TRY(write_cbor_output_string(s, false)); - } - g_query.incremental_match_slice(s.ptr, s.len); - } else { - return "main: internal error: unexpected string-token conversion"; - } - - if (continued) { - return nullptr; - } - - if (g_flags.output_format == file_format::json) { - if (!(vbd & WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8)) { - TRY(write_json_output_byte_string(wuffs_base__empty_slice_u8(), true)); - } - TRY(write_dst("\"", 1)); - } else { - TRY(write_cbor_output_string(wuffs_base__empty_slice_u8(), true)); - } - return nullptr; + return write_dst(&u[0], n); } // ---- @@ -1565,23 +913,16 @@ handle_token(wuffs_base__token t, bool start_of_token_chain) { if (g_query.matched_all() && (g_depth >= g_flags.max_output_depth)) { g_suppress_write_dst--; // '…' is U+2026 HORIZONTAL ELLIPSIS, which is 3 UTF-8 bytes. - if (g_flags.output_format == file_format::json) { - TRY(write_dst((vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST) - ? "\"[…]\"" - : "\"{…}\"", - 7)); - } else { - TRY(write_dst((vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST) - ? "\x65[…]" - : "\x65{…}", - 6)); - } - } else if (g_flags.output_format == file_format::json) { + TRY(write_dst((vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST) + ? "\"[…]\"" + : "\"{…}\"", + 7)); + } else { // Write preceding whitespace. if ((g_ctx != context::in_list_after_bracket) && (g_ctx != context::in_dict_after_brace) && !g_flags.compact_output) { - if (g_flags.output_json_extra_comma) { + if (g_flags.output_extra_comma) { TRY(write_dst(",\n", 2)); } else { TRY(write_dst("\n", 1)); @@ -1596,8 +937,6 @@ handle_token(wuffs_base__token t, bool start_of_token_chain) { TRY(write_dst( (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__FROM_LIST) ? "]" : "}", 1)); - } else { - TRY(write_dst("\xFF", 1)); } g_ctx = (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST) @@ -1607,31 +946,13 @@ handle_token(wuffs_base__token t, bool start_of_token_chain) { } // Write preceding whitespace and punctuation, if it wasn't ']', '}' or a - // continuation of a multi-token chain or a CBOR tagged data item. - if (g_previous_token_was_cbor_tag) { - g_previous_token_was_cbor_tag = false; - } else if (start_of_token_chain) { - if (g_flags.output_format != file_format::json) { - // No-op. - } else if (g_ctx == context::in_dict_after_key) { + // continuation of a multi-token chain. + if (start_of_token_chain) { + if (g_ctx == context::in_dict_after_key) { TRY(write_dst(": ", g_flags.compact_output ? 1 : 2)); } else if (g_ctx != context::none) { - if ((g_ctx == context::in_dict_after_brace) || - (g_ctx == context::in_dict_after_value)) { - // Reject dict keys that aren't UTF-8 strings or non-negative - // integers, which could otherwise happen with -i=cbor -o=json. - if (vbc == WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_UNSIGNED) { - // No-op. - } else if ((vbc == WUFFS_BASE__TOKEN__VBC__STRING) && - (vbd & - WUFFS_BASE__TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8)) { - // No-op. - } else { - return "main: cannot convert CBOR non-text-string to JSON map key"; - } - } - if ((g_ctx == context::in_list_after_value) || - (g_ctx == context::in_dict_after_value)) { + if ((g_ctx != context::in_list_after_bracket) && + (g_ctx != context::in_dict_after_brace)) { TRY(write_dst(",", 1)); } if (!g_flags.compact_output) { @@ -1688,15 +1009,10 @@ handle_token(wuffs_base__token t, bool start_of_token_chain) { case WUFFS_BASE__TOKEN__VBC__STRUCTURE: if (g_query.matched_all() && (g_depth >= g_flags.max_output_depth)) { g_suppress_write_dst++; - } else if (g_flags.output_format == file_format::json) { + } else { TRY(write_dst( (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST) ? "[" : "{", 1)); - } else { - TRY(write_dst((vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST) - ? "\x9F" - : "\xBF", - 1)); } g_depth++; g_ctx = (vbd & WUFFS_BASE__TOKEN__VBD__STRUCTURE__TO_LIST) @@ -1705,10 +1021,26 @@ handle_token(wuffs_base__token t, bool start_of_token_chain) { return nullptr; case WUFFS_BASE__TOKEN__VBC__STRING: - TRY(handle_string(vbd, tok, start_of_token_chain, t.continued())); + if (start_of_token_chain) { + TRY(write_dst("\"", 1)); + g_query.restart_fragment(in_dict_before_key() && + g_query.is_at(g_depth)); + } + + if (vbd & WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP) { + // No-op. + } else if (vbd & + WUFFS_BASE__TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY) { + TRY(write_dst(tok.ptr, tok.len)); + g_query.incremental_match_slice(tok.ptr, tok.len); + } else { + return "main: internal error: unexpected string-token conversion"; + } + if (t.continued()) { return nullptr; } + TRY(write_dst("\"", 1)); goto after_value; case WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT: @@ -1720,81 +1052,12 @@ handle_token(wuffs_base__token t, bool start_of_token_chain) { return nullptr; case WUFFS_BASE__TOKEN__VBC__LITERAL: - TRY(write_literal(vbd)); - goto after_value; - case WUFFS_BASE__TOKEN__VBC__NUMBER: - TRY(write_number(vbd, tok)); + TRY(write_dst(tok.ptr, tok.len)); goto after_value; - - case WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_SIGNED: - case WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_UNSIGNED: { - bool x_is_signed = vbc == WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_SIGNED; - uint64_t x = x_is_signed - ? ((uint64_t)(t.value_base_detail__sign_extended())) - : vbd; - if (t.continued()) { - if (tok.len != 0) { - return "main: internal error: unexpected to-be-extended length"; - } - g_token_extension.category = vbc; - g_token_extension.detail = x; - return nullptr; - } - TRY(write_inline_integer(x, x_is_signed, tok)); - goto after_value; - } } - int64_t ext = t.value_extension(); - if (ext >= 0) { - uint64_t x = (g_token_extension.detail - << WUFFS_BASE__TOKEN__VALUE_EXTENSION__NUM_BITS) | - ((uint64_t)ext); - switch (g_token_extension.category) { - case WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_SIGNED: - case WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_UNSIGNED: - TRY(write_inline_integer( - x, - g_token_extension.category == - WUFFS_BASE__TOKEN__VBC__INLINE_INTEGER_SIGNED, - tok)); - g_token_extension.category = 0; - g_token_extension.detail = 0; - goto after_value; - case CATEGORY_CBOR_TAG: - g_previous_token_was_cbor_tag = true; - TRY(write_cbor_tag(x, tok)); - g_token_extension.category = 0; - g_token_extension.detail = 0; - return nullptr; - } - } - - if (t.value_major() == WUFFS_CBOR__TOKEN_VALUE_MAJOR) { - uint64_t value_minor = t.value_minor(); - if (value_minor & WUFFS_CBOR__TOKEN_VALUE_MINOR__MINUS_1_MINUS_X) { - TRY(write_cbor_minus_1_minus_x(tok)); - goto after_value; - } else if (value_minor & WUFFS_CBOR__TOKEN_VALUE_MINOR__SIMPLE_VALUE) { - TRY(write_cbor_simple_value(vbd, tok)); - goto after_value; - } else if (value_minor & WUFFS_CBOR__TOKEN_VALUE_MINOR__TAG) { - g_previous_token_was_cbor_tag = true; - if (t.continued()) { - if (tok.len != 0) { - return "main: internal error: unexpected to-be-extended length"; - } - g_token_extension.category = CATEGORY_CBOR_TAG; - g_token_extension.detail = vbd; - return nullptr; - } - return write_cbor_tag(vbd, tok); - } - } - - // Return an error if we didn't match the (value_major, value_minor) or - // (vbc, vbd) pair. + // Return an error if we didn't match the (vbc, vbd) pair. return "main: internal error: unexpected token"; } while (0); @@ -1830,7 +1093,7 @@ main1(int argc, char** argv) { bool start_of_token_chain = true; while (true) { - wuffs_base__status status = g_dec->decode_tokens( + wuffs_base__status status = g_dec.decode_tokens( &g_tok, &g_src, wuffs_base__make_slice_u8(g_work_buffer_array, WORK_BUFFER_ARRAY_SIZE)); @@ -1885,7 +1148,7 @@ end_of_data: TRY(read_src()); } if ((g_src.meta.ri < g_src.meta.wi) || !g_src.meta.closed) { - return "main: valid JSON|CBOR followed by further (unexpected) data"; + return "main: valid JSON followed by further (unexpected) data"; } // Check that we've used all of the decoded tokens, other than trailing @@ -1965,9 +1228,7 @@ main(int argc, char** argv) { const char* z = main1(argc, argv); if (g_wrote_to_dst) { - const char* z1 = (g_flags.output_format == file_format::json) - ? write_dst("\n", 1) - : nullptr; + const char* z1 = write_dst("\n", 1); const char* z2 = flush_dst(); z = z ? z : (z1 ? z1 : z2); }