// Copyright 2020 The Wuffs Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // https://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. pub struct decoder? implements base.token_decoder( // quirk_enabled_allow_backslash_etc, an 8-element array, is indexed by the // same enum as lut_quirky_backslashes. quirk_enabled_allow_backslash_etc : array[8] base.bool, quirk_enabled_allow_backslash_capital_u : base.bool, quirk_enabled_allow_backslash_x : base.bool, quirk_enabled_allow_comment_block : base.bool, quirk_enabled_allow_comment_line : base.bool, quirk_enabled_allow_extra_comma : base.bool, quirk_enabled_allow_inf_nan_numbers : base.bool, quirk_enabled_allow_leading_ascii_record_separator : base.bool, quirk_enabled_allow_leading_unicode_byte_order_mark : base.bool, quirk_enabled_allow_trailing_new_line : base.bool, quirk_enabled_replace_invalid_utf_8 : base.bool, allow_leading_ars : base.bool, allow_leading_ubom : base.bool, end_of_data : base.bool, )( // stack is conceptually an array of bits, implemented as an array of u32. // The N'th bit being 0 or 1 means that we're in an array or object, where // N is the recursion depth. // // Parsing JSON involves recursion: containers (arrays and objects) can // hold other containers. As child elements are completed, the parser needs // to remember 1 bit of state per recursion depth: whether the parent // container was an array or an object. When continuing to parse the // parent's elements, `, "key": value` is only valid for objects. // // Note that we explicitly track our own stack and depth. We do not use the // call stack to hold this state and the decoder.decode_tokens function is // not recursive per se. // // Wuffs code does not have the capability to dynamically allocate memory, // so the maximum depth is hard-coded at compile time. In this case, the // maximum is 1024 (stack is 1024 bits or 128 bytes), also known as // decoder_depth_max_incl. // // The [JSON spec](https://www.ietf.org/rfc/rfc8259.txt) clearly states, // "an implementation may set limits on the maximum depth of nesting". // // In comparison, as of February 2020, the Chromium web browser's JSON // parser's maximum recursion depth is 200: // https://source.chromium.org/chromium/chromium/src/+/3dece34cde622faa0daac07156c25d92c9897d1e:base/json/json_common.h;l=18 // // Other languages and libraries' maximum depths (determined empirically) // are listed at https://github.com/lovasoa/bad_json_parsers#results stack : array[1024 / 32] base.u32, ) pub func decoder.set_quirk_enabled!(quirk: base.u32, enabled: base.bool) { if args.quirk == quirk_allow_backslash_a { this.quirk_enabled_allow_backslash_etc[1] = args.enabled } else if args.quirk == quirk_allow_backslash_capital_u { this.quirk_enabled_allow_backslash_capital_u = args.enabled } else if args.quirk == quirk_allow_backslash_e { this.quirk_enabled_allow_backslash_etc[2] = args.enabled } else if args.quirk == quirk_allow_backslash_question_mark { this.quirk_enabled_allow_backslash_etc[3] = args.enabled } else if args.quirk == quirk_allow_backslash_single_quote { this.quirk_enabled_allow_backslash_etc[4] = args.enabled } else if args.quirk == quirk_allow_backslash_v { this.quirk_enabled_allow_backslash_etc[5] = args.enabled } else if args.quirk == quirk_allow_backslash_x { this.quirk_enabled_allow_backslash_x = args.enabled } else if args.quirk == quirk_allow_backslash_zero { this.quirk_enabled_allow_backslash_etc[6] = args.enabled } else if args.quirk == quirk_allow_comment_block { this.quirk_enabled_allow_comment_block = args.enabled } else if args.quirk == quirk_allow_comment_line { this.quirk_enabled_allow_comment_line = args.enabled } else if args.quirk == quirk_allow_extra_comma { this.quirk_enabled_allow_extra_comma = args.enabled } else if args.quirk == quirk_allow_inf_nan_numbers { this.quirk_enabled_allow_inf_nan_numbers = args.enabled } else if args.quirk == quirk_allow_leading_ascii_record_separator { this.quirk_enabled_allow_leading_ascii_record_separator = args.enabled } else if args.quirk == quirk_allow_leading_unicode_byte_order_mark { this.quirk_enabled_allow_leading_unicode_byte_order_mark = args.enabled } else if args.quirk == quirk_allow_trailing_new_line { this.quirk_enabled_allow_trailing_new_line = args.enabled } else if args.quirk == quirk_replace_invalid_utf_8 { this.quirk_enabled_replace_invalid_utf_8 = args.enabled } } pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) { var vminor : base.u32[..= 0xFF_FFFF] var number_length : base.u32[..= 0x3FF] var number_status : base.u32[..= 0x3] var string_length : base.u32[..= 0xFFFB] var whitespace_length : base.u32[..= 0xFFFE] var depth : base.u32[..= 1024] var stack_byte : base.u32[..= (1024 / 32) - 1] var stack_bit : base.u32[..= 31] var match : base.u32[..= 2] var c4 : base.u32 var c : base.u8 var backslash : base.u8 var char : base.u8 var class : base.u8[..= 0x0F] var multi_byte_utf8 : base.u32 var backslash_x_length : base.u32[..= 0xFFFF] var backslash_x_ok : base.u8 var backslash_x_string : base.u32 var uni4_ok : base.u8 var uni4_string : base.u64 var uni4_value : base.u32[..= 0xFFFF] var uni4_high_surrogate : base.u32[..= 0x10_FC00] var uni8_ok : base.u8 var uni8_string : base.u64 var uni8_value : base.u32[..= 0xFFFF_FFFF] // expect is a bitmask of what the next character class can be. // // expect_after_value is what to expect after seeing a value (a literal, // number, string, array or object). For depth 0, this is ignored. // Otherwise, it should be (EXPECT_CLOSE_FOO | EXPECT_COMMA), for some // value of FOO. var expect : base.u32 var expect_after_value : base.u32 while this.end_of_data { return base."@end of data" } if this.quirk_enabled_allow_leading_ascii_record_separator or this.quirk_enabled_allow_leading_unicode_byte_order_mark { this.decode_leading?(dst: args.dst, src: args.src) } expect = 0x1EB2 // EXPECT_VALUE while.outer true { while.goto_parsed_a_leaf_value true { if args.dst.available() <= 0 { yield? base."$short write" continue.outer } // Consume whitespace. whitespace_length = 0 c = 0 class = 0 while.ws true, inv args.dst.available() > 0, post args.src.available() > 0, { if args.src.available() <= 0 { if whitespace_length > 0 { args.dst.write_fast_token!( value_major: 0, value_minor: 0, link: 0x0, length: whitespace_length) whitespace_length = 0 } if args.src.is_closed() { return "#bad input" } yield? base."$short read" whitespace_length = 0 continue.outer } c = args.src.peek_u8() class = lut_classes[c] if class <> 0x00 { // 0x00 is CLASS_WHITESPACE. break.ws } args.src.skip32_fast!(actual: 1, worst_case: 1) if whitespace_length >= 0xFFFE { args.dst.write_fast_token!( value_major: 0, value_minor: 0, link: 0x0, length: 0xFFFF) whitespace_length = 0 continue.outer } whitespace_length += 1 } endwhile.ws // Emit whitespace. if whitespace_length > 0 { args.dst.write_fast_token!( value_major: 0, value_minor: 0, link: 0x0, length: whitespace_length) whitespace_length = 0 if args.dst.available() <= 0 { continue.outer } } // Check expected character classes. if 0 == (expect & ((1 as base.u32) << class)) { return "#bad input" } // These assertions are redundant (the Wuffs compiler should // already know these facts; deleting these assertions should still // compile) but are listed explicitly to guard against future edits // to the code above inadvertently invalidating these assertions. assert args.dst.available() > 0 assert args.src.available() > 0 if class == 0x01 { // 0x01 is CLASS_STRING. // -------- BEGIN parse strings. // Emit the leading '"'. args.dst.write_fast_token!( value_major: 0, value_minor: 0x40_0013, link: 0x1, length: 1) args.src.skip32_fast!(actual: 1, worst_case: 1) while.string_loop_outer true { if args.dst.available() <= 0 { yield? base."$short write" continue.string_loop_outer } string_length = 0 while.string_loop_inner true, pre args.dst.available() > 0, { if args.src.available() <= 0 { if string_length > 0 { args.dst.write_fast_token!( value_major: 0, value_minor: 0x40_0021, link: 0x3, length: string_length) string_length = 0 } if args.src.is_closed() { return "#bad input" } yield? base."$short read" string_length = 0 continue.string_loop_outer } // As an optimization, consume non-special ASCII 4 bytes at a time. while args.src.available() > 4, inv args.dst.available() > 0, inv args.src.available() > 0, { c4 = args.src.peek_u32le() if 0x00 <> (lut_chars[0xFF & (c4 >> 0)] | lut_chars[0xFF & (c4 >> 8)] | lut_chars[0xFF & (c4 >> 16)] | lut_chars[0xFF & (c4 >> 24)]) { break } args.src.skip32_fast!(actual: 4, worst_case: 4) if string_length > (0xFFFB - 4) { args.dst.write_fast_token!( value_major: 0, value_minor: 0x40_0021, link: 0x3, length: string_length + 4) string_length = 0 continue.string_loop_outer } string_length += 4 } c = args.src.peek_u8() char = lut_chars[c] if char == 0x00 { // Non-special ASCII. args.src.skip32_fast!(actual: 1, worst_case: 1) if string_length >= 0xFFFB { args.dst.write_fast_token!( value_major: 0, value_minor: 0x40_0021, link: 0x3, length: 0xFFFC) string_length = 0 continue.string_loop_outer } string_length += 1 continue.string_loop_inner } else if char == 0x01 { // '"' if string_length <> 0 { args.dst.write_fast_token!( value_major: 0, value_minor: 0x40_0021, link: 0x3, length: string_length) string_length = 0 } break.string_loop_outer } else if char == 0x02 { // '\\'. if string_length > 0 { args.dst.write_fast_token!( value_major: 0, value_minor: 0x40_0021, link: 0x3, length: string_length) string_length = 0 if args.dst.available() <= 0 { continue.string_loop_outer } } assert args.dst.available() > 0 if args.src.available() < 2 { if args.src.is_closed() { return "#bad backslash-escape" } yield? base."$short read" string_length = 0 char = 0 continue.string_loop_outer } c = (args.src.peek_u16le() >> 8) as base.u8 backslash = lut_backslashes[c] if (backslash & 0x80) <> 0 { args.src.skip32_fast!(actual: 2, worst_case: 2) args.dst.write_fast_token!( value_major: 0, value_minor: 0x60_0000 | ((backslash & 0x7F) as base.u32), link: 0x3, length: 2) continue.string_loop_outer } else if backslash <> 0 { if this.quirk_enabled_allow_backslash_etc[backslash & 7] { args.src.skip32_fast!(actual: 2, worst_case: 2) args.dst.write_fast_token!( value_major: 0, value_minor: 0x60_0000 | (lut_quirky_backslashes[backslash & 7] as base.u32), link: 0x3, length: 2) continue.string_loop_outer } } else if c == 0x75 { // 0x75 is 'u'. // -------- BEGIN backslash-u. if args.src.available() < 6 { if args.src.is_closed() { return "#bad backslash-escape" } yield? base."$short read" string_length = 0 char = 0 continue.string_loop_outer } uni4_string = args.src.peek_u48le_as_u64() >> 16 uni4_value = 0 uni4_ok = 0x80 c = lut_hexadecimal_digits[0xFF & (uni4_string >> 0)] uni4_ok &= c uni4_value |= ((c & 0x0F) as base.u32) << 12 c = lut_hexadecimal_digits[0xFF & (uni4_string >> 8)] uni4_ok &= c uni4_value |= ((c & 0x0F) as base.u32) << 8 c = lut_hexadecimal_digits[0xFF & (uni4_string >> 16)] uni4_ok &= c uni4_value |= ((c & 0x0F) as base.u32) << 4 c = lut_hexadecimal_digits[0xFF & (uni4_string >> 24)] uni4_ok &= c uni4_value |= ((c & 0x0F) as base.u32) << 0 if uni4_ok == 0 { // It wasn't 4 hexadecimal digits. No-op // (and fall through to "#bad // backslash-escape"). } else if (uni4_value < 0xD800) or (0xDFFF < uni4_value) { // Not a Unicode surrogate. We're good. args.src.skip32_fast!(actual: 6, worst_case: 6) args.dst.write_fast_token!( value_major: 0, value_minor: 0x60_0000 | uni4_value, link: 0x3, length: 6) continue.string_loop_outer } else if uni4_value >= 0xDC00 { // Low surrogate. No-op (and fall through // to "#bad backslash-escape"). } else { // High surrogate, which needs to be // followed by a "\\u1234" low surrogate. // We've already peeked 6 bytes for the // high surrogate. We need 12 in total: // another 8 bytes at an offset of 4. if args.src.available() < 12 { if args.src.is_closed() { if this.quirk_enabled_replace_invalid_utf_8 { args.src.skip32_fast!(actual: 6, worst_case: 6) args.dst.write_fast_token!( value_major: 0, value_minor: 0x60_FFFD, link: 0x3, length: 6) continue.string_loop_outer } return "#bad backslash-escape" } yield? base."$short read" string_length = 0 uni4_value = 0 char = 0 continue.string_loop_outer } uni4_string = args.src.peek_u64le_at(offset: 4) >> 16 // Look for the low surrogate's "\\u". if ((0xFF & (uni4_string >> 0)) <> 0x5C) or ((0xFF & (uni4_string >> 8)) <> 0x75) { uni4_high_surrogate = 0 uni4_value = 0 uni4_ok = 0 } else { uni4_high_surrogate = 0x1_0000 + ((uni4_value - 0xD800) << 10) uni4_value = 0 uni4_ok = 0x80 uni4_string >>= 16 c = lut_hexadecimal_digits[0xFF & (uni4_string >> 0)] uni4_ok &= c uni4_value |= ((c & 0x0F) as base.u32) << 12 c = lut_hexadecimal_digits[0xFF & (uni4_string >> 8)] uni4_ok &= c uni4_value |= ((c & 0x0F) as base.u32) << 8 c = lut_hexadecimal_digits[0xFF & (uni4_string >> 16)] uni4_ok &= c uni4_value |= ((c & 0x0F) as base.u32) << 4 c = lut_hexadecimal_digits[0xFF & (uni4_string >> 24)] uni4_ok &= c uni4_value |= ((c & 0x0F) as base.u32) << 0 } if (uni4_ok <> 0) and (0xDC00 <= uni4_value) and (uni4_value <= 0xDFFF) { // Emit a single token for the surrogate pair. uni4_value -= 0xDC00 args.src.skip32_fast!(actual: 12, worst_case: 12) args.dst.write_fast_token!( value_major: 0, value_minor: 0x60_0000 | uni4_high_surrogate | uni4_value, link: 0x3, length: 12) continue.string_loop_outer } } if this.quirk_enabled_replace_invalid_utf_8 { if args.src.available() < 6 { return "#internal error: inconsistent I/O" } args.src.skip32_fast!(actual: 6, worst_case: 6) args.dst.write_fast_token!( value_major: 0, value_minor: 0x60_FFFD, link: 0x3, length: 6) continue.string_loop_outer } // -------- END backslash-u. } else if (c == 0x55) and this.quirk_enabled_allow_backslash_capital_u { // 0x55 is 'U'. // -------- BEGIN backslash-capital-u. if args.src.available() < 10 { if args.src.is_closed() { return "#bad backslash-escape" } yield? base."$short read" string_length = 0 char = 0 continue.string_loop_outer } uni8_string = args.src.peek_u64le_at(offset: 2) uni8_value = 0 uni8_ok = 0x80 c = lut_hexadecimal_digits[0xFF & (uni8_string >> 0)] uni8_ok &= c uni8_value |= ((c & 0x0F) as base.u32) << 28 c = lut_hexadecimal_digits[0xFF & (uni8_string >> 8)] uni8_ok &= c uni8_value |= ((c & 0x0F) as base.u32) << 24 c = lut_hexadecimal_digits[0xFF & (uni8_string >> 16)] uni8_ok &= c uni8_value |= ((c & 0x0F) as base.u32) << 20 c = lut_hexadecimal_digits[0xFF & (uni8_string >> 24)] uni8_ok &= c uni8_value |= ((c & 0x0F) as base.u32) << 16 c = lut_hexadecimal_digits[0xFF & (uni8_string >> 32)] uni8_ok &= c uni8_value |= ((c & 0x0F) as base.u32) << 12 c = lut_hexadecimal_digits[0xFF & (uni8_string >> 40)] uni8_ok &= c uni8_value |= ((c & 0x0F) as base.u32) << 8 c = lut_hexadecimal_digits[0xFF & (uni8_string >> 48)] uni8_ok &= c uni8_value |= ((c & 0x0F) as base.u32) << 4 c = lut_hexadecimal_digits[0xFF & (uni8_string >> 56)] uni8_ok &= c uni8_value |= ((c & 0x0F) as base.u32) << 0 if uni8_ok == 0 { // It wasn't 8 hexadecimal digits. No-op // (and fall through to "#bad // backslash-escape"). } else if (uni8_value < 0xD800) or ( (0xDFFF < uni8_value) and (uni8_value <= 0x10_FFFF)) { // Not a Unicode surrogate. We're good. args.src.skip32_fast!(actual: 10, worst_case: 10) args.dst.write_fast_token!( value_major: 0, value_minor: 0x60_0000 | (uni8_value & 0x1F_FFFF), link: 0x3, length: 10) continue.string_loop_outer } else if this.quirk_enabled_replace_invalid_utf_8 { args.src.skip32_fast!(actual: 10, worst_case: 10) args.dst.write_fast_token!( value_major: 0, value_minor: 0x60_FFFD, link: 0x3, length: 6) continue.string_loop_outer } // -------- END backslash-capital-u. } else if (c == 0x78) and this.quirk_enabled_allow_backslash_x { // 0x78 is 'x'. // -------- BEGIN backslash-x if args.src.available() < 4 { if args.src.is_closed() { return "#bad backslash-escape" } yield? base."$short read" string_length = 0 char = 0 continue.string_loop_outer } backslash_x_length = 0 while (backslash_x_length <= 0xFFFB) and (args.src.available() >= 4), inv args.dst.available() > 0, { backslash_x_string = args.src.peek_u32le() backslash_x_ok = 0x80 c = lut_hexadecimal_digits[0xFF & (backslash_x_string >> 16)] backslash_x_ok &= c c = lut_hexadecimal_digits[0xFF & (backslash_x_string >> 24)] backslash_x_ok &= c if (backslash_x_ok == 0) or ((backslash_x_string & 0xFFFF) <> 0x785C) { // It wasn't "\\x34", for some // hexadecimal digits "34". break } args.src.skip32_fast!(actual: 4, worst_case: 4) backslash_x_length += 4 } if backslash_x_length == 0 { return "#bad backslash-escape" } args.dst.write_fast_token!( value_major: 0, value_minor: 0x40_0080, link: 0x3, length: backslash_x_length) continue.string_loop_outer // -------- END backslash-x } return "#bad backslash-escape" } else if char == 0x03 { // 2-byte UTF-8. if args.src.available() < 2 { if string_length > 0 { args.dst.write_fast_token!( value_major: 0, value_minor: 0x40_0021, link: 0x3, length: string_length) string_length = 0 if args.dst.available() <= 0 { continue.string_loop_outer } } if args.src.is_closed() { if this.quirk_enabled_replace_invalid_utf_8 { args.dst.write_fast_token!( value_major: 0, value_minor: 0x60_FFFD, link: 0x3, length: 1) args.src.skip32_fast!(actual: 1, worst_case: 1) continue.string_loop_inner } return "#bad UTF-8" } yield? base."$short read" string_length = 0 char = 0 continue.string_loop_outer } multi_byte_utf8 = args.src.peek_u16le_as_u32() if (multi_byte_utf8 & 0xC000) == 0x8000 { multi_byte_utf8 = (0x00_07C0 & (multi_byte_utf8 ~mod<< 6)) | (0x00_003F & (multi_byte_utf8 >> 8)) args.src.skip32_fast!(actual: 2, worst_case: 2) if string_length >= 0xFFF8 { args.dst.write_fast_token!( value_major: 0, value_minor: 0x40_0021, link: 0x3, length: string_length + 2) string_length = 0 continue.string_loop_outer } string_length += 2 continue.string_loop_inner } } else if char == 0x04 { // 3-byte UTF-8. if args.src.available() < 3 { if string_length > 0 { args.dst.write_fast_token!( value_major: 0, value_minor: 0x40_0021, link: 0x3, length: string_length) string_length = 0 if args.dst.available() <= 0 { continue.string_loop_outer } } if args.src.is_closed() { if this.quirk_enabled_replace_invalid_utf_8 { args.dst.write_fast_token!( value_major: 0, value_minor: 0x60_FFFD, link: 0x3, length: 1) args.src.skip32_fast!(actual: 1, worst_case: 1) continue.string_loop_inner } return "#bad UTF-8" } yield? base."$short read" string_length = 0 char = 0 continue.string_loop_outer } multi_byte_utf8 = args.src.peek_u24le_as_u32() if (multi_byte_utf8 & 0xC0_C000) == 0x80_8000 { multi_byte_utf8 = (0x00_F000 & (multi_byte_utf8 ~mod<< 12)) | (0x00_0FC0 & (multi_byte_utf8 >> 2)) | (0x00_003F & (multi_byte_utf8 >> 16)) if (0x07FF < multi_byte_utf8) and ((multi_byte_utf8 < 0xD800) or (0xDFFF < multi_byte_utf8)) { args.src.skip32_fast!(actual: 3, worst_case: 3) if string_length >= 0xFFF8 { args.dst.write_fast_token!( value_major: 0, value_minor: 0x40_0021, link: 0x3, length: string_length + 3) string_length = 0 continue.string_loop_outer } string_length += 3 continue.string_loop_inner } } } else if char == 0x05 { // 4-byte UTF-8. if args.src.available() < 4 { if string_length > 0 { args.dst.write_fast_token!( value_major: 0, value_minor: 0x40_0021, link: 0x3, length: string_length) string_length = 0 if args.dst.available() <= 0 { continue.string_loop_outer } } if args.src.is_closed() { if this.quirk_enabled_replace_invalid_utf_8 { args.dst.write_fast_token!( value_major: 0, value_minor: 0x60_FFFD, link: 0x3, length: 1) args.src.skip32_fast!(actual: 1, worst_case: 1) continue.string_loop_inner } return "#bad UTF-8" } yield? base."$short read" string_length = 0 char = 0 continue.string_loop_outer } multi_byte_utf8 = args.src.peek_u32le() if (multi_byte_utf8 & 0xC0C0_C000) == 0x8080_8000 { multi_byte_utf8 = (0x1C_0000 & (multi_byte_utf8 ~mod<< 18)) | (0x03_F000 & (multi_byte_utf8 ~mod<< 4)) | (0x00_0FC0 & (multi_byte_utf8 >> 10)) | (0x00_003F & (multi_byte_utf8 >> 24)) if (0xFFFF < multi_byte_utf8) and (multi_byte_utf8 <= 0x10_FFFF) { args.src.skip32_fast!(actual: 4, worst_case: 4) if string_length >= 0xFFF8 { args.dst.write_fast_token!( value_major: 0, value_minor: 0x40_0021, link: 0x3, length: string_length + 4) string_length = 0 continue.string_loop_outer } string_length += 4 continue.string_loop_inner } } } if string_length > 0 { args.dst.write_fast_token!( value_major: 0, value_minor: 0x40_0021, link: 0x3, length: string_length) string_length = 0 if args.dst.available() <= 0 { continue.string_loop_outer } } if char == 0x80 { return "#bad C0 control code" } if this.quirk_enabled_replace_invalid_utf_8 { args.dst.write_fast_token!( value_major: 0, value_minor: 0x60_FFFD, link: 0x3, length: 1) args.src.skip32_fast!(actual: 1, worst_case: 1) continue.string_loop_inner } return "#bad UTF-8" } endwhile.string_loop_inner } endwhile.string_loop_outer // Emit the trailing '"'. while true { if args.src.available() <= 0 { if args.src.is_closed() { return "#bad input" } yield? base."$short read" continue } if args.dst.available() <= 0 { yield? base."$short write" continue } args.src.skip32_fast!(actual: 1, worst_case: 1) args.dst.write_fast_token!( value_major: 0, value_minor: 0x40_0013, link: 0x2, length: 1) break } // As above, expect must have contained EXPECT_STRING. If it // didn't also contain EXPECT_NUMBER then we were parsing an // object key and the next token should be ':'. if 0 == (expect & 0x0010) { // 0x0010 is (1 << CLASS_NUMBER). expect = 0x1008 // 0x1008 is EXPECT_COLON. continue.outer } break.goto_parsed_a_leaf_value // -------- END parse strings. } else if class == 0x02 { // 0x02 is CLASS_COMMA. args.src.skip32_fast!(actual: 1, worst_case: 1) // The ',' is filler. args.dst.write_fast_token!( value_major: 0, value_minor: 0, link: 0x0, length: 1) // What's valid after a comma depends on whether or not we're // in an array or an object. if 0 == (expect & 0x0100) { // 0x0100 is (1 << CLASS_CLOSE_SQUARE_BRACKET). if this.quirk_enabled_allow_extra_comma { expect = 0x1042 // 0x1042 is EXPECT_STRING | EXPECT_CLOSE_CURLY_BRACE. } else { expect = 0x1002 // 0x1002 is EXPECT_STRING. } } else { if this.quirk_enabled_allow_extra_comma { expect = 0x1FB2 // 0x0FB2 is EXPECT_VALUE | EXPECT_CLOSE_SQUARE_BRACKET. } else { expect = 0x1EB2 // 0x0EB2 is EXPECT_VALUE. } } continue.outer } else if class == 0x03 { // 0x03 is CLASS_COLON. args.src.skip32_fast!(actual: 1, worst_case: 1) // The ':' is filler. args.dst.write_fast_token!( value_major: 0, value_minor: 0, link: 0x0, length: 1) expect = 0x1EB2 // 0x1EB2 is EXPECT_VALUE. continue.outer } else if class == 0x04 { // 0x04 is CLASS_NUMBER. // -------- BEGIN parse numbers. while true, pre args.dst.available() > 0, { number_length = this.decode_number!(src: args.src) number_status = number_length >> 8 vminor = 0xA0_0403 if (number_length & 0x80) <> 0 { vminor = 0xA0_0401 } number_length = number_length & 0x7F if number_status == 0 { args.dst.write_fast_token!( value_major: 0, value_minor: vminor, link: 0x0, length: number_length) break } while number_length > 0 { number_length -= 1 if args.src.can_undo_byte() { args.src.undo_byte!() } else { return "#internal error: inconsistent I/O" } } if number_status == 1 { if this.quirk_enabled_allow_inf_nan_numbers { this.decode_inf_nan?(dst: args.dst, src: args.src) break } return "#bad input" } else if number_status == 2 { return "#unsupported number length" } else { yield? base."$short read" while args.dst.available() <= 0, post args.dst.available() > 0, { yield? base."$short write" } } } break.goto_parsed_a_leaf_value // -------- END parse numbers. } else if class == 0x05 { // 0x05 is CLASS_OPEN_CURLY_BRACE. vminor = 0x20_4011 if depth == 0 { // No-op. } else if 0 <> (expect_after_value & 0x0040) { // 0x0040 is (1 << CLASS_CLOSE_CURLY_BRACE). vminor = 0x20_4041 } else { vminor = 0x20_4021 } if depth >= 1024 { return "#unsupported recursion depth" } stack_byte = depth / 32 stack_bit = depth & 31 this.stack[stack_byte] |= (1 as base.u32) << stack_bit depth += 1 args.src.skip32_fast!(actual: 1, worst_case: 1) args.dst.write_fast_token!( value_major: 0, value_minor: vminor, link: 0x0, length: 1) expect = 0x1042 // 0x1042 is (EXPECT_CLOSE_CURLY_BRACE | EXPECT_STRING). expect_after_value = 0x1044 // 0x1044 is (EXPECT_CURLY_CLOSE_BRACE | EXPECT_COMMA). continue.outer } else if class == 0x06 { // 0x06 is CLASS_CLOSE_CURLY_BRACE. args.src.skip32_fast!(actual: 1, worst_case: 1) if depth <= 1 { args.dst.write_fast_token!( value_major: 0, value_minor: 0x20_1042, link: 0x0, length: 1) break.outer } depth -= 1 stack_byte = (depth - 1) / 32 stack_bit = (depth - 1) & 31 if 0 == (this.stack[stack_byte] & ((1 as base.u32) << stack_bit)) { args.dst.write_fast_token!( value_major: 0, value_minor: 0x20_2042, link: 0x0, length: 1) // 0x1104 is (EXPECT_SQUARE_CLOSE_BRACKET | EXPECT_COMMA). expect = 0x1104 expect_after_value = 0x1104 } else { args.dst.write_fast_token!( value_major: 0, value_minor: 0x20_4042, link: 0x0, length: 1) // 0x1044 is (EXPECT_CURLY_CLOSE_BRACE | EXPECT_COMMA). expect = 0x1044 expect_after_value = 0x1044 } continue.outer } else if class == 0x07 { // 0x07 is CLASS_OPEN_SQUARE_BRACKET. vminor = 0x20_2011 if depth == 0 { // No-op. } else if 0 <> (expect_after_value & 0x0040) { // 0x0040 is (1 << CLASS_CLOSE_CURLY_BRACE). vminor = 0x20_2041 } else { vminor = 0x20_2021 } if depth >= 1024 { return "#unsupported recursion depth" } stack_byte = depth / 32 stack_bit = depth & 31 this.stack[stack_byte] &= 0xFFFF_FFFF ^ ((1 as base.u32) << stack_bit) depth += 1 args.src.skip32_fast!(actual: 1, worst_case: 1) args.dst.write_fast_token!( value_major: 0, value_minor: vminor, link: 0x0, length: 1) expect = 0x1FB2 // 0x1FB2 is (EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_VALUE). expect_after_value = 0x1104 // 0x1104 is (EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA). continue.outer } else if class == 0x08 { // 0x08 is CLASS_CLOSE_SQUARE_BRACKET. args.src.skip32_fast!(actual: 1, worst_case: 1) if depth <= 1 { args.dst.write_fast_token!( value_major: 0, value_minor: 0x20_1022, link: 0x0, length: 1) break.outer } depth -= 1 stack_byte = (depth - 1) / 32 stack_bit = (depth - 1) & 31 if 0 == (this.stack[stack_byte] & ((1 as base.u32) << stack_bit)) { args.dst.write_fast_token!( value_major: 0, value_minor: 0x20_2022, link: 0x0, length: 1) // 0x1104 is (EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA). expect = 0x1104 expect_after_value = 0x1104 } else { args.dst.write_fast_token!( value_major: 0, value_minor: 0x20_4022, link: 0x0, length: 1) // 0x1044 is (EXPECT_CLOSE_CURLY_BRACE | EXPECT_COMMA). expect = 0x1044 expect_after_value = 0x1044 } continue.outer } else if class == 0x09 { // 0x09 is CLASS_FALSE. match = args.src.match7(a: 0x6573_6C61_6605) // 5 bytes "false". if match == 0 { args.dst.write_fast_token!( value_major: 0, value_minor: 0x80_0004, link: 0x0, length: 5) if args.src.available() < 5 { return "#internal error: inconsistent I/O" } args.src.skip32_fast!(actual: 5, worst_case: 5) break.goto_parsed_a_leaf_value } else if match == 1 { yield? base."$short read" continue.outer } } else if class == 0x0A { // 0x0A is CLASS_TRUE. match = args.src.match7(a: 0x65_7572_7404) // 4 bytes "true". if match == 0 { args.dst.write_fast_token!( value_major: 0, value_minor: 0x80_0008, link: 0x0, length: 4) if args.src.available() < 4 { return "#internal error: inconsistent I/O" } args.src.skip32_fast!(actual: 4, worst_case: 4) break.goto_parsed_a_leaf_value } else if match == 1 { yield? base."$short read" continue.outer } } else if class == 0x0B { // 0x0B is CLASS_NULL_NAN_INF. match = args.src.match7(a: 0x6C_6C75_6E04) // 4 bytes "null". if match == 0 { args.dst.write_fast_token!( value_major: 0, value_minor: 0x80_0002, link: 0x0, length: 4) if args.src.available() < 4 { return "#internal error: inconsistent I/O" } args.src.skip32_fast!(actual: 4, worst_case: 4) break.goto_parsed_a_leaf_value } else if match == 1 { yield? base."$short read" continue.outer } if this.quirk_enabled_allow_inf_nan_numbers { this.decode_inf_nan?(dst: args.dst, src: args.src) break.goto_parsed_a_leaf_value } } else if class == 0x0C { // 0x0C is CLASS_COMMENT. if this.quirk_enabled_allow_comment_block or this.quirk_enabled_allow_comment_line { this.decode_comment?(dst: args.dst, src: args.src) continue.outer } } return "#bad input" } endwhile.goto_parsed_a_leaf_value // We've just parsed a leaf (non-container) value: literal (null, // false, true), number or string. if depth == 0 { break.outer } expect = expect_after_value } endwhile.outer if this.quirk_enabled_allow_trailing_new_line { this.decode_trailing_new_line?(dst: args.dst, src: args.src) } this.end_of_data = true } pri func decoder.decode_number!(src: base.io_reader) base.u32[..= 0x3FF] { var c : base.u8 var n : base.u32[..= 0x3FF] var floating_point : base.u32[..= 0x80] while.goto_done true { n = 0 // Peek. if args.src.available() <= 0 { if not args.src.is_closed() { n |= 0x300 } break.goto_done } c = args.src.peek_u8() // Scan the optional minus sign. if c <> 0x2D { // 0x2D is '-'. assert args.src.available() > 0 assert n <= 1 } else { n += 1 args.src.skip32_fast!(actual: 1, worst_case: 1) // Peek. if args.src.available() <= 0 { if not args.src.is_closed() { n |= 0x300 } n |= 0x100 // A '-' without digits is invalid. break.goto_done } c = args.src.peek_u8() assert args.src.available() > 0 assert n <= 1 } // Scan the opening digits. if c == 0x30 { // 0x30 is '0'. n += 1 args.src.skip32_fast!(actual: 1, worst_case: 1) assert n <= 99 } else { n = this.decode_digits!(src: args.src, n: n) if n > 99 { break.goto_done } assert n <= 99 } // Peek. if args.src.available() <= 0 { if not args.src.is_closed() { n |= 0x300 } break.goto_done } c = args.src.peek_u8() // Scan the optional fraction. if c <> 0x2E { // 0x2E is '.'. assert args.src.available() > 0 assert n <= 99 } else { if n >= 99 { n |= 0x200 break.goto_done } n += 1 args.src.skip32_fast!(actual: 1, worst_case: 1) floating_point = 0x80 n = this.decode_digits!(src: args.src, n: n) if n > 99 { break.goto_done } // Peek. if args.src.available() <= 0 { if not args.src.is_closed() { n |= 0x300 } break.goto_done } c = args.src.peek_u8() assert args.src.available() > 0 assert n <= 99 } // Scan the optional 'E' or 'e'. if (c <> 0x45) and (c <> 0x65) { // 0x45 and 0x65 are 'E' and 'e'. break.goto_done } if n >= 99 { n |= 0x200 break.goto_done } n += 1 args.src.skip32_fast!(actual: 1, worst_case: 1) floating_point = 0x80 assert n <= 99 // Peek. if args.src.available() <= 0 { if not args.src.is_closed() { n |= 0x300 } n |= 0x100 // An 'E' or 'e' without digits is invalid. break.goto_done } c = args.src.peek_u8() // Scan the optional '+' or '-'. if (c <> 0x2B) and (c <> 0x2D) { // 0x2B and 0x2D are '+' and '-'. assert n <= 99 } else { if n >= 99 { n |= 0x200 break.goto_done } n += 1 args.src.skip32_fast!(actual: 1, worst_case: 1) assert n <= 99 } // Scan the exponent digits. n = this.decode_digits!(src: args.src, n: n) break.goto_done } endwhile.goto_done return n | floating_point } pri func decoder.decode_digits!(src: base.io_reader, n: base.u32[..= 99]) base.u32[..= 0x3FF] { var c : base.u8 var n : base.u32[..= 0x3FF] n = args.n while true { if args.src.available() <= 0 { if not args.src.is_closed() { n |= 0x300 } break } c = args.src.peek_u8() if 0x00 == lut_decimal_digits[c] { break } // Cap decoder_number_length_max_incl at an arbitrary value, 99. The // caller's src.data.len should therefore be at least 100, also known // as decoder_src_io_buffer_length_min_incl. // // An example of a JSON number that is 81 bytes long is: // https://github.com/nst/JSONTestSuite/blob/master/test_parsing/y_number_double_close_to_zero.json // // Note that 99 (in hex, 0x63) is less than 0x80, so we can use 0x80 as // a flag bit in func decoder.decode_number. if n >= 99 { n |= 0x200 break } n += 1 args.src.skip32_fast!(actual: 1, worst_case: 1) } if n == args.n { n |= 0x100 } return n } pri func decoder.decode_leading?(dst: base.token_writer, src: base.io_reader) { var c : base.u8 var u : base.u32 this.allow_leading_ars = this.quirk_enabled_allow_leading_ascii_record_separator this.allow_leading_ubom = this.quirk_enabled_allow_leading_unicode_byte_order_mark while this.allow_leading_ars or this.allow_leading_ubom { if args.dst.available() <= 0 { yield? base."$short write" continue } if args.src.available() <= 0 { if args.src.is_closed() { break } yield? base."$short read" continue } c = args.src.peek_u8() if (c == 0x1E) and this.allow_leading_ars { this.allow_leading_ars = false args.src.skip32_fast!(actual: 1, worst_case: 1) args.dst.write_fast_token!( value_major: 0, value_minor: 0, link: 0x0, length: 1) continue } else if (c == 0xEF) and this.allow_leading_ubom { if args.src.available() < 3 { if args.src.is_closed() { break } yield? base."$short read" continue } u = args.src.peek_u24le_as_u32() if u == 0xBF_BBEF { this.allow_leading_ubom = false args.src.skip32_fast!(actual: 3, worst_case: 3) args.dst.write_fast_token!( value_major: 0, value_minor: 0, link: 0x0, length: 3) continue } } break } } pri func decoder.decode_comment?(dst: base.token_writer, src: base.io_reader) { var c : base.u8 var c2 : base.u16 var link_prev : base.u32[..= 0x2] var length : base.u32[..= 0xFFFD] while (args.dst.available() <= 0) or (args.src.available() <= 1), post args.dst.available() > 0, post args.src.available() > 1, { if args.dst.available() <= 0 { yield? base."$short write" continue } if args.src.is_closed() { return "#bad input" } yield? base."$short read" } c2 = args.src.peek_u16le() if (c2 == 0x2A2F) and this.quirk_enabled_allow_comment_block { args.src.skip32_fast!(actual: 2, worst_case: 2) length = 2 while.comment_block true { if args.dst.available() <= 0 { yield? base."$short write" length = 0 continue.comment_block } while true, pre args.dst.available() > 0, { if args.src.available() <= 1 { if length > 0 { args.dst.write_fast_token!( value_major: 0, value_minor: 0x2, link: 0x1 | link_prev, length: length) link_prev = 0x2 } if args.src.is_closed() { return "#bad input" } yield? base."$short read" length = 0 continue.comment_block } c2 = args.src.peek_u16le() if c2 == 0x2F2A { // 0x2F2A is "*/" little-endian. args.src.skip32_fast!(actual: 2, worst_case: 2) args.dst.write_fast_token!( value_major: 0, value_minor: 0x2, link: link_prev, length: length + 2) return ok } args.src.skip32_fast!(actual: 1, worst_case: 1) if length >= 0xFFFD { args.dst.write_fast_token!( value_major: 0, value_minor: 0x2, link: 0x1 | link_prev, length: length + 1) length = 0 link_prev = 0x2 continue.comment_block } length += 1 } } endwhile.comment_block } else if (c2 == 0x2F2F) and this.quirk_enabled_allow_comment_line { args.src.skip32_fast!(actual: 2, worst_case: 2) length = 2 while.comment_line true { if args.dst.available() <= 0 { yield? base."$short write" length = 0 continue.comment_line } while true, pre args.dst.available() > 0, { if args.src.available() <= 0 { if length > 0 { args.dst.write_fast_token!( value_major: 0, value_minor: 0x1, link: 0x1 | link_prev, length: length) link_prev = 0x2 } if args.src.is_closed() { return "#bad input" } yield? base."$short read" length = 0 continue.comment_line } c = args.src.peek_u8() if c == 0x0A { // 0x0A is '\n'. args.src.skip32_fast!(actual: 1, worst_case: 1) args.dst.write_fast_token!( value_major: 0, value_minor: 0x1, link: link_prev, length: length + 1) return ok } args.src.skip32_fast!(actual: 1, worst_case: 1) if length >= 0xFFFD { args.dst.write_fast_token!( value_major: 0, value_minor: 0x1, link: 0x1 | link_prev, length: length + 1) length = 0 link_prev = 0x2 continue.comment_line } length += 1 } } endwhile.comment_line } return "#bad input" } pri func decoder.decode_inf_nan?(dst: base.token_writer, src: base.io_reader) { var c4 : base.u32 var neg : base.u32[..= 1] while true { if args.dst.available() <= 0 { yield? base."$short write" continue } if args.src.available() <= 2 { if args.src.is_closed() { return "#bad input" } yield? base."$short read" continue } // Bitwise or'ing with 0x20 converts upper case ASCII to lower case. c4 = args.src.peek_u24le_as_u32() if (c4 | 0x20_2020) == 0x66_6E69 { if args.src.available() > 7 { if (args.src.peek_u64le() | 0x2020_2020_2020_2020) == 0x7974_696E_6966_6E69 { args.dst.write_fast_token!( value_major: 0, value_minor: 0xA0_0020, link: 0x0, length: 8) args.src.skip32_fast!(actual: 8, worst_case: 8) return ok } } else if not args.src.is_closed() { yield? base."$short read" continue } args.dst.write_fast_token!( value_major: 0, value_minor: 0xA0_0020, link: 0x0, length: 3) args.src.skip32_fast!(actual: 3, worst_case: 3) return ok } else if (c4 | 0x20_2020) == 0x6E_616E { args.dst.write_fast_token!( value_major: 0, value_minor: 0xA0_0080, link: 0x0, length: 3) args.src.skip32_fast!(actual: 3, worst_case: 3) return ok } else if (c4 & 0xFF) == 0x2B { // 0x2B is '+'. neg = 0 } else if (c4 & 0xFF) == 0x2D { // 0x2D is '-'. neg = 1 } else { return "#bad input" } if args.src.available() <= 3 { if args.src.is_closed() { return "#bad input" } yield? base."$short read" continue } c4 = args.src.peek_u32le() >> 8 if (c4 | 0x20_2020) == 0x66_6E69 { if args.src.available() > 8 { if (args.src.peek_u64le_at(offset: 1) | 0x2020_2020_2020_2020) == 0x7974_696E_6966_6E69 { args.dst.write_fast_token!( value_major: 0, value_minor: 0xA0_0000 | ((0x20 as base.u32) >> neg), link: 0x0, length: 9) args.src.skip32_fast!(actual: 9, worst_case: 9) return ok } } else if not args.src.is_closed() { yield? base."$short read" continue } args.dst.write_fast_token!( value_major: 0, value_minor: 0xA0_0000 | ((0x20 as base.u32) >> neg), link: 0x0, length: 4) args.src.skip32_fast!(actual: 4, worst_case: 4) return ok } else if (c4 | 0x20_2020) == 0x6E_616E { args.dst.write_fast_token!( value_major: 0, value_minor: 0xA0_0000 | ((0x80 as base.u32) >> neg), link: 0x0, length: 4) args.src.skip32_fast!(actual: 4, worst_case: 4) return ok } return "#bad input" } } pri func decoder.decode_trailing_new_line?(dst: base.token_writer, src: base.io_reader) { var c : base.u8 var whitespace_length : base.u32[..= 0xFFFE] while.outer true { if args.dst.available() <= 0 { yield? base."$short write" whitespace_length = 0 continue.outer } while.inner true, pre args.dst.available() > 0, { if args.src.available() <= 0 { if whitespace_length > 0 { args.dst.write_fast_token!( value_major: 0, value_minor: 0, link: 0x0, length: whitespace_length) whitespace_length = 0 } if args.src.is_closed() { break.outer } yield? base."$short read" whitespace_length = 0 continue.outer } c = args.src.peek_u8() if lut_classes[c] <> 0x00 { // 0x00 is CLASS_WHITESPACE. if whitespace_length > 0 { args.dst.write_fast_token!( value_major: 0, value_minor: 0, link: 0x0, length: whitespace_length) whitespace_length = 0 } return "#bad input" } args.src.skip32_fast!(actual: 1, worst_case: 1) if (whitespace_length >= 0xFFFE) or (c == 0x0A) { args.dst.write_fast_token!( value_major: 0, value_minor: 0, link: 0x0, length: whitespace_length + 1) whitespace_length = 0 if c == 0x0A { break.outer } continue.outer } whitespace_length += 1 } endwhile.inner } endwhile.outer }