mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-17 16:32:11 +01:00
1761 lines
76 KiB
Plaintext
1761 lines
76 KiB
Plaintext
// Copyright 2020 The Wuffs Authors.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// https://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
pub struct decoder? implements base.token_decoder(
|
|
quirks : array[QUIRKS_COUNT] base.bool,
|
|
|
|
allow_leading_ars : base.bool,
|
|
allow_leading_ubom : base.bool,
|
|
|
|
end_of_data : base.bool,
|
|
|
|
trailer_stop : base.u8,
|
|
|
|
// comment_type is set as a side-effect of decode_comment?.
|
|
// - 0 means no comment.
|
|
// - 1 means a block comment.
|
|
// - 2 means a line comment.
|
|
comment_type : base.u8,
|
|
|
|
util : base.utility,
|
|
) + (
|
|
// stack is conceptually an array of bits, implemented as an array of u32.
|
|
// The N'th bit being 0 or 1 means that we're in an array or object, where
|
|
// N is the recursion depth.
|
|
//
|
|
// Parsing JSON involves recursion: containers (arrays and objects) can
|
|
// hold other containers. As child elements are completed, the parser needs
|
|
// to remember 1 bit of state per recursion depth: whether the parent
|
|
// container was an array or an object. When continuing to parse the
|
|
// parent's elements, `, "key": value` is only valid for objects.
|
|
//
|
|
// Note that we explicitly track our own stack and depth. We do not use the
|
|
// call stack to hold this state and the decoder.decode_tokens function is
|
|
// not recursive per se.
|
|
//
|
|
// Wuffs code does not have the capability to dynamically allocate memory,
|
|
// so the maximum depth is hard-coded at compile time. In this case, the
|
|
// maximum is 1024 (stack is 1024 bits or 128 bytes), also known as
|
|
// DECODER_DEPTH_MAX_INCL.
|
|
//
|
|
// The [JSON spec](https://www.ietf.org/rfc/rfc8259.txt) clearly states,
|
|
// "an implementation may set limits on the maximum depth of nesting".
|
|
//
|
|
// In comparison, as of February 2020, the Chromium web browser's JSON
|
|
// parser's maximum recursion depth is 200:
|
|
// https://source.chromium.org/chromium/chromium/src/+/3dece34cde622faa0daac07156c25d92c9897d1e:base/json/json_common.h;l=18
|
|
//
|
|
// Other languages and libraries' maximum depths (determined empirically)
|
|
// are listed at https://github.com/lovasoa/bad_json_parsers#results
|
|
stack : array[1024 / 32] base.u32,
|
|
)
|
|
|
|
pub func decoder.get_quirk(key: base.u32) base.u64 {
|
|
var key : base.u32
|
|
|
|
if args.key >= QUIRKS_BASE {
|
|
key = args.key - QUIRKS_BASE
|
|
if key < QUIRKS_COUNT {
|
|
if this.quirks[key] {
|
|
return 1
|
|
}
|
|
}
|
|
}
|
|
return 0
|
|
}
|
|
|
|
pub func decoder.set_quirk!(key: base.u32, value: base.u64) base.status {
|
|
if args.key >= QUIRKS_BASE {
|
|
args.key -= QUIRKS_BASE
|
|
if args.key < QUIRKS_COUNT {
|
|
this.quirks[args.key] = args.value > 0
|
|
return ok
|
|
}
|
|
}
|
|
return base."#unsupported option"
|
|
}
|
|
|
|
pub func decoder.history_retain_length() base.u64 {
|
|
return 0
|
|
}
|
|
|
|
pub func decoder.workbuf_len() base.range_ii_u64 {
|
|
return this.util.empty_range_ii_u64()
|
|
}
|
|
|
|
pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, workbuf: slice base.u8) {
|
|
// This is a very, very long function, and it is tempting to refactor it.
|
|
// Be careful of performance impacts when doing so. For example, commit
|
|
// 86d3b89f "Factor out json.decoder.decode_string" pulled out a 500 line
|
|
// decode_string function, which was certainly cleaner structurally, but
|
|
// also regressed performance by 1.1x to 1.2x. For details, see
|
|
// https://github.com/google/wuffs/commit/86d3b89f9a6578d964a4b6d71e21dfc9bb702b44
|
|
|
|
var vminor : base.u32[..= 0xFF_FFFF]
|
|
var number_length : base.u32[..= 0x3FF]
|
|
var number_status : base.u32[..= 0x3]
|
|
var string_length : base.u32[..= 0xFFFB]
|
|
var whitespace_length : base.u32[..= 0xFFFE]
|
|
var depth : base.u32[..= 1024]
|
|
var stack_byte : base.u32[..= (1024 / 32) - 1]
|
|
var stack_bit : base.u32[..= 31]
|
|
var match : base.u32[..= 2]
|
|
var c4 : base.u32
|
|
var c : base.u8
|
|
var backslash : base.u8
|
|
var char : base.u8
|
|
var class : base.u8[..= 0x0F]
|
|
var multi_byte_utf8 : base.u32
|
|
|
|
var backslash_x_ok : base.u8
|
|
var backslash_x_value : base.u8
|
|
var backslash_x_string : base.u32
|
|
|
|
var uni4_ok : base.u8
|
|
var uni4_string : base.u64
|
|
var uni4_value : base.u32[..= 0xFFFF]
|
|
var uni4_high_surrogate : base.u32[..= 0x10_FC00]
|
|
|
|
var uni8_ok : base.u8
|
|
var uni8_string : base.u64
|
|
var uni8_value : base.u32[..= 0xFFFF_FFFF]
|
|
|
|
// expect is a bitmask of what the next character class can be.
|
|
//
|
|
// expect_after_value is what to expect after seeing a value (a literal,
|
|
// number, string, array or object). For depth 0, this is ignored.
|
|
// Otherwise, it should be (EXPECT_CLOSE_FOO | EXPECT_COMMA), for some
|
|
// value of FOO.
|
|
var expect : base.u32
|
|
var expect_after_value : base.u32
|
|
|
|
if this.end_of_data {
|
|
return base."@end of data"
|
|
}
|
|
|
|
if this.quirks[QUIRK_EXPECT_TRAILING_NEW_LINE_OR_EOF - QUIRKS_BASE] {
|
|
if this.quirks[QUIRK_ALLOW_COMMENT_BLOCK - QUIRKS_BASE] or
|
|
this.quirks[QUIRK_ALLOW_COMMENT_LINE - QUIRKS_BASE] or
|
|
this.quirks[QUIRK_ALLOW_TRAILING_FILLER - QUIRKS_BASE] {
|
|
return "#bad quirk combination"
|
|
}
|
|
}
|
|
|
|
if this.quirks[QUIRK_ALLOW_LEADING_ASCII_RECORD_SEPARATOR - QUIRKS_BASE] or
|
|
this.quirks[QUIRK_ALLOW_LEADING_UNICODE_BYTE_ORDER_MARK - QUIRKS_BASE] {
|
|
this.decode_leading?(dst: args.dst, src: args.src)
|
|
}
|
|
|
|
expect = EXPECT_VALUE
|
|
|
|
while.outer true {
|
|
while.goto_parsed_a_leaf_value true {{
|
|
if args.dst.length() <= 0 {
|
|
yield? base."$short write"
|
|
continue.outer
|
|
}
|
|
|
|
// Consume whitespace.
|
|
whitespace_length = 0
|
|
c = 0
|
|
class = 0
|
|
while.ws true,
|
|
inv args.dst.length() > 0,
|
|
post args.src.length() > 0,
|
|
{
|
|
if args.src.length() <= 0 {
|
|
if whitespace_length > 0 {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: 0,
|
|
continued: 0,
|
|
length: whitespace_length)
|
|
whitespace_length = 0
|
|
}
|
|
if args.src.is_closed() {
|
|
return "#bad input"
|
|
}
|
|
yield? base."$short read"
|
|
continue.outer
|
|
}
|
|
|
|
c = args.src.peek_u8()
|
|
class = LUT_CLASSES[c]
|
|
if class <> CLASS_WHITESPACE {
|
|
break.ws
|
|
}
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
|
|
if whitespace_length >= 0xFFFE {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: 0,
|
|
continued: 0,
|
|
length: 0xFFFF)
|
|
whitespace_length = 0
|
|
continue.outer
|
|
}
|
|
whitespace_length += 1
|
|
} endwhile.ws
|
|
|
|
// Emit whitespace.
|
|
if whitespace_length > 0 {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: 0,
|
|
continued: 0,
|
|
length: whitespace_length)
|
|
whitespace_length = 0
|
|
if args.dst.length() <= 0 {
|
|
continue.outer
|
|
}
|
|
}
|
|
|
|
// Check expected character classes.
|
|
if 0 == (expect & ((1 as base.u32) << class)) {
|
|
return "#bad input"
|
|
}
|
|
|
|
// These assertions are redundant (the Wuffs compiler should already
|
|
// know these facts; deleting these assertions should still compile)
|
|
// but are listed explicitly to guard against future edits to the code
|
|
// above inadvertently invalidating these assertions.
|
|
assert args.dst.length() > 0
|
|
assert args.src.length() > 0
|
|
|
|
if class == CLASS_STRING {
|
|
// -------- BEGIN parse strings.
|
|
// Emit the leading '"'.
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
|
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
|
base.TOKEN__VBD__STRING__DEFINITELY_ASCII |
|
|
base.TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP,
|
|
continued: 1,
|
|
length: 1)
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
|
|
while.string_loop_outer true {
|
|
if args.dst.length() <= 0 {
|
|
yield? base."$short write"
|
|
continue.string_loop_outer
|
|
}
|
|
|
|
string_length = 0
|
|
while.string_loop_inner true,
|
|
pre args.dst.length() > 0,
|
|
{
|
|
if args.src.length() <= 0 {
|
|
if string_length > 0 {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
|
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
|
continued: 1,
|
|
length: string_length)
|
|
string_length = 0
|
|
}
|
|
if args.src.is_closed() {
|
|
return "#bad input"
|
|
}
|
|
yield? base."$short read"
|
|
continue.string_loop_outer
|
|
}
|
|
|
|
// As an optimization, consume non-special ASCII 4 bytes at
|
|
// a time.
|
|
while args.src.length() > 4,
|
|
inv args.dst.length() > 0,
|
|
inv args.src.length() > 0,
|
|
{
|
|
c4 = args.src.peek_u32le()
|
|
if 0x00 <> (LUT_CHARS[0xFF & (c4 >> 0)] |
|
|
LUT_CHARS[0xFF & (c4 >> 8)] |
|
|
LUT_CHARS[0xFF & (c4 >> 16)] |
|
|
LUT_CHARS[0xFF & (c4 >> 24)]) {
|
|
break
|
|
}
|
|
args.src.skip_u32_fast!(actual: 4, worst_case: 4)
|
|
if string_length > (0xFFFB - 4) {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
|
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
|
continued: 1,
|
|
length: string_length + 4)
|
|
string_length = 0
|
|
continue.string_loop_outer
|
|
}
|
|
string_length += 4
|
|
} endwhile
|
|
|
|
c = args.src.peek_u8()
|
|
char = LUT_CHARS[c]
|
|
|
|
if char == 0x00 { // Non-special ASCII.
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
if string_length >= 0xFFFB {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
|
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
|
continued: 1,
|
|
length: 0xFFFC)
|
|
string_length = 0
|
|
continue.string_loop_outer
|
|
}
|
|
string_length += 1
|
|
continue.string_loop_inner
|
|
|
|
} else if char == 0x01 { // '"'
|
|
if string_length <> 0 {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
|
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
|
continued: 1,
|
|
length: string_length)
|
|
string_length = 0
|
|
}
|
|
break.string_loop_outer
|
|
|
|
} else if char == 0x02 { // '\\'.
|
|
if string_length > 0 {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
|
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
|
continued: 1,
|
|
length: string_length)
|
|
string_length = 0
|
|
if args.dst.length() <= 0 {
|
|
continue.string_loop_outer
|
|
}
|
|
}
|
|
assert args.dst.length() > 0
|
|
|
|
if args.src.length() < 2 {
|
|
if args.src.is_closed() {
|
|
return "#bad backslash-escape"
|
|
}
|
|
yield? base."$short read"
|
|
continue.string_loop_outer
|
|
}
|
|
c = (args.src.peek_u16le() >> 8) as base.u8
|
|
backslash = LUT_BACKSLASHES[c]
|
|
if (backslash & 0x80) <> 0 {
|
|
args.src.skip_u32_fast!(actual: 2, worst_case: 2)
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) |
|
|
((backslash & 0x7F) as base.u32),
|
|
continued: 1,
|
|
length: 2)
|
|
continue.string_loop_outer
|
|
|
|
} else if backslash <> 0 {
|
|
if this.quirks[LUT_QUIRKY_BACKSLASHES_QUIRKS[backslash & 7]] {
|
|
args.src.skip_u32_fast!(actual: 2, worst_case: 2)
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) |
|
|
(LUT_QUIRKY_BACKSLASHES_CHARS[backslash & 7] as base.u32),
|
|
continued: 1,
|
|
length: 2)
|
|
continue.string_loop_outer
|
|
}
|
|
|
|
} else if c == 'u' {
|
|
// -------- BEGIN backslash-u.
|
|
if args.src.length() < 6 {
|
|
if args.src.is_closed() {
|
|
return "#bad backslash-escape"
|
|
}
|
|
yield? base."$short read"
|
|
continue.string_loop_outer
|
|
}
|
|
|
|
uni4_string = args.src.peek_u48le_as_u64() >> 16
|
|
uni4_value = 0
|
|
uni4_ok = 0x80
|
|
|
|
c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni4_string >> 0)]
|
|
uni4_ok &= c
|
|
uni4_value |= ((c & 0x0F) as base.u32) << 12
|
|
c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni4_string >> 8)]
|
|
uni4_ok &= c
|
|
uni4_value |= ((c & 0x0F) as base.u32) << 8
|
|
c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni4_string >> 16)]
|
|
uni4_ok &= c
|
|
uni4_value |= ((c & 0x0F) as base.u32) << 4
|
|
c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni4_string >> 24)]
|
|
uni4_ok &= c
|
|
uni4_value |= ((c & 0x0F) as base.u32) << 0
|
|
|
|
if uni4_ok == 0 {
|
|
// It wasn't 4 hexadecimal digits. No-op (and
|
|
// fall through to "#bad backslash-escape").
|
|
|
|
} else if (uni4_value < 0xD800) or (0xDFFF < uni4_value) {
|
|
// Not a Unicode surrogate. We're good.
|
|
args.src.skip_u32_fast!(actual: 6, worst_case: 6)
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) |
|
|
uni4_value,
|
|
continued: 1,
|
|
length: 6)
|
|
continue.string_loop_outer
|
|
|
|
} else if uni4_value >= 0xDC00 {
|
|
// Low surrogate. No-op (and fall through to
|
|
// "#bad backslash-escape").
|
|
|
|
} else {
|
|
// High surrogate, which needs to be followed
|
|
// by a "\\u1234" low surrogate. We've already
|
|
// peeked 6 bytes for the high surrogate. We
|
|
// need 12 in total: another 8 bytes at an
|
|
// offset of 4.
|
|
if args.src.length() < 12 {
|
|
if args.src.is_closed() {
|
|
if this.quirks[QUIRK_REPLACE_INVALID_UNICODE - QUIRKS_BASE] {
|
|
args.src.skip_u32_fast!(actual: 6, worst_case: 6)
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) |
|
|
base.UNICODE__REPLACEMENT_CHARACTER,
|
|
continued: 1,
|
|
length: 6)
|
|
continue.string_loop_outer
|
|
}
|
|
return "#bad backslash-escape"
|
|
}
|
|
yield? base."$short read"
|
|
continue.string_loop_outer
|
|
}
|
|
uni4_string = args.src.peek_u64le_at(offset: 4) >> 16
|
|
|
|
// Look for the low surrogate's "\\u".
|
|
if ((0xFF & (uni4_string >> 0)) <> '\\') or
|
|
((0xFF & (uni4_string >> 8)) <> 'u') {
|
|
uni4_high_surrogate = 0
|
|
uni4_value = 0
|
|
uni4_ok = 0
|
|
} else {
|
|
uni4_high_surrogate =
|
|
0x1_0000 + ((uni4_value - 0xD800) << 10)
|
|
uni4_value = 0
|
|
uni4_ok = 0x80
|
|
uni4_string >>= 16
|
|
|
|
c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni4_string >> 0)]
|
|
uni4_ok &= c
|
|
uni4_value |= ((c & 0x0F) as base.u32) << 12
|
|
c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni4_string >> 8)]
|
|
uni4_ok &= c
|
|
uni4_value |= ((c & 0x0F) as base.u32) << 8
|
|
c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni4_string >> 16)]
|
|
uni4_ok &= c
|
|
uni4_value |= ((c & 0x0F) as base.u32) << 4
|
|
c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni4_string >> 24)]
|
|
uni4_ok &= c
|
|
uni4_value |= ((c & 0x0F) as base.u32) << 0
|
|
}
|
|
|
|
if (uni4_ok <> 0) and
|
|
(0xDC00 <= uni4_value) and (uni4_value <= 0xDFFF) {
|
|
|
|
// Emit a single token for the surrogate
|
|
// pair.
|
|
uni4_value -= 0xDC00
|
|
args.src.skip_u32_fast!(actual: 12, worst_case: 12)
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) |
|
|
uni4_high_surrogate | uni4_value,
|
|
continued: 1,
|
|
length: 12)
|
|
continue.string_loop_outer
|
|
}
|
|
}
|
|
|
|
if this.quirks[QUIRK_REPLACE_INVALID_UNICODE - QUIRKS_BASE] {
|
|
if args.src.length() < 6 {
|
|
return "#internal error: inconsistent I/O"
|
|
}
|
|
args.src.skip_u32_fast!(actual: 6, worst_case: 6)
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) |
|
|
base.UNICODE__REPLACEMENT_CHARACTER,
|
|
continued: 1,
|
|
length: 6)
|
|
continue.string_loop_outer
|
|
}
|
|
// -------- END backslash-u.
|
|
|
|
} else if (c == 'U') and
|
|
this.quirks[QUIRK_ALLOW_BACKSLASH_CAPITAL_U - QUIRKS_BASE] {
|
|
// -------- BEGIN backslash-capital-u.
|
|
if args.src.length() < 10 {
|
|
if args.src.is_closed() {
|
|
return "#bad backslash-escape"
|
|
}
|
|
yield? base."$short read"
|
|
continue.string_loop_outer
|
|
}
|
|
uni8_string = args.src.peek_u64le_at(offset: 2)
|
|
uni8_value = 0
|
|
uni8_ok = 0x80
|
|
|
|
c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni8_string >> 0)]
|
|
uni8_ok &= c
|
|
uni8_value |= ((c & 0x0F) as base.u32) << 28
|
|
c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni8_string >> 8)]
|
|
uni8_ok &= c
|
|
uni8_value |= ((c & 0x0F) as base.u32) << 24
|
|
c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni8_string >> 16)]
|
|
uni8_ok &= c
|
|
uni8_value |= ((c & 0x0F) as base.u32) << 20
|
|
c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni8_string >> 24)]
|
|
uni8_ok &= c
|
|
uni8_value |= ((c & 0x0F) as base.u32) << 16
|
|
c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni8_string >> 32)]
|
|
uni8_ok &= c
|
|
uni8_value |= ((c & 0x0F) as base.u32) << 12
|
|
c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni8_string >> 40)]
|
|
uni8_ok &= c
|
|
uni8_value |= ((c & 0x0F) as base.u32) << 8
|
|
c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni8_string >> 48)]
|
|
uni8_ok &= c
|
|
uni8_value |= ((c & 0x0F) as base.u32) << 4
|
|
c = LUT_HEXADECIMAL_DIGITS[0xFF & (uni8_string >> 56)]
|
|
uni8_ok &= c
|
|
uni8_value |= ((c & 0x0F) as base.u32) << 0
|
|
|
|
if uni8_ok == 0 {
|
|
// It wasn't 8 hexadecimal digits. No-op (and
|
|
// fall through to "#bad backslash-escape").
|
|
|
|
} else if (uni8_value < 0xD800) or (
|
|
(0xDFFF < uni8_value) and (uni8_value <= 0x10_FFFF)) {
|
|
// Not a Unicode surrogate. We're good.
|
|
args.src.skip_u32_fast!(actual: 10, worst_case: 10)
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) |
|
|
(uni8_value & 0x1F_FFFF),
|
|
continued: 1,
|
|
length: 10)
|
|
continue.string_loop_outer
|
|
} else if this.quirks[QUIRK_REPLACE_INVALID_UNICODE - QUIRKS_BASE] {
|
|
args.src.skip_u32_fast!(actual: 10, worst_case: 10)
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) |
|
|
base.UNICODE__REPLACEMENT_CHARACTER,
|
|
continued: 1,
|
|
length: 10)
|
|
continue.string_loop_outer
|
|
}
|
|
// -------- END backslash-capital-u.
|
|
|
|
} else if (c == 'x') and
|
|
this.quirks[QUIRK_ALLOW_BACKSLASH_X_AS_CODE_POINTS - QUIRKS_BASE] {
|
|
// -------- BEGIN backslash-x
|
|
if args.src.length() < 4 {
|
|
if args.src.is_closed() {
|
|
return "#bad backslash-escape"
|
|
}
|
|
yield? base."$short read"
|
|
continue.string_loop_outer
|
|
}
|
|
|
|
backslash_x_string = args.src.peek_u32le()
|
|
backslash_x_ok = 0x80
|
|
|
|
c = LUT_HEXADECIMAL_DIGITS[0xFF & (backslash_x_string >> 16)]
|
|
backslash_x_ok &= c
|
|
backslash_x_value = ((c & 0x0F) << 4) as base.u8
|
|
c = LUT_HEXADECIMAL_DIGITS[0xFF & (backslash_x_string >> 24)]
|
|
backslash_x_ok &= c
|
|
backslash_x_value = (backslash_x_value | (c & 0x0F)) as base.u8
|
|
|
|
if (backslash_x_ok == 0) or
|
|
((backslash_x_string & 0xFFFF) <> 0x785C) {
|
|
// It wasn't "\\x34", for some hexadecimal
|
|
// digits "34".
|
|
return "#bad backslash-escape"
|
|
}
|
|
args.src.skip_u32_fast!(actual: 4, worst_case: 4)
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) |
|
|
(backslash_x_value as base.u32),
|
|
continued: 1,
|
|
length: 4)
|
|
continue.string_loop_outer
|
|
// -------- END backslash-x
|
|
}
|
|
|
|
return "#bad backslash-escape"
|
|
|
|
} else if char == 0x03 { // 2-byte UTF-8.
|
|
if args.src.length() < 2 {
|
|
if string_length > 0 {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
|
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
|
continued: 1,
|
|
length: string_length)
|
|
string_length = 0
|
|
if args.dst.length() <= 0 {
|
|
continue.string_loop_outer
|
|
}
|
|
}
|
|
if args.src.is_closed() {
|
|
if this.quirks[QUIRK_REPLACE_INVALID_UNICODE - QUIRKS_BASE] {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) |
|
|
base.UNICODE__REPLACEMENT_CHARACTER,
|
|
continued: 1,
|
|
length: 1)
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
continue.string_loop_outer
|
|
}
|
|
return "#bad UTF-8"
|
|
}
|
|
yield? base."$short read"
|
|
continue.string_loop_outer
|
|
}
|
|
multi_byte_utf8 = args.src.peek_u16le_as_u32()
|
|
if (multi_byte_utf8 & 0xC000) == 0x8000 {
|
|
multi_byte_utf8 = (0x00_07C0 & (multi_byte_utf8 ~mod<< 6)) |
|
|
(0x00_003F & (multi_byte_utf8 >> 8))
|
|
args.src.skip_u32_fast!(actual: 2, worst_case: 2)
|
|
if string_length >= 0xFFF8 {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
|
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
|
continued: 1,
|
|
length: string_length + 2)
|
|
string_length = 0
|
|
continue.string_loop_outer
|
|
}
|
|
string_length += 2
|
|
continue.string_loop_inner
|
|
}
|
|
|
|
} else if char == 0x04 { // 3-byte UTF-8.
|
|
if args.src.length() < 3 {
|
|
if string_length > 0 {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
|
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
|
continued: 1,
|
|
length: string_length)
|
|
string_length = 0
|
|
if args.dst.length() <= 0 {
|
|
continue.string_loop_outer
|
|
}
|
|
}
|
|
if args.src.is_closed() {
|
|
if this.quirks[QUIRK_REPLACE_INVALID_UNICODE - QUIRKS_BASE] {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) |
|
|
base.UNICODE__REPLACEMENT_CHARACTER,
|
|
continued: 1,
|
|
length: 1)
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
continue.string_loop_outer
|
|
}
|
|
return "#bad UTF-8"
|
|
}
|
|
yield? base."$short read"
|
|
continue.string_loop_outer
|
|
}
|
|
multi_byte_utf8 = args.src.peek_u24le_as_u32()
|
|
if (multi_byte_utf8 & 0xC0_C000) == 0x80_8000 {
|
|
multi_byte_utf8 = (0x00_F000 & (multi_byte_utf8 ~mod<< 12)) |
|
|
(0x00_0FC0 & (multi_byte_utf8 >> 2)) |
|
|
(0x00_003F & (multi_byte_utf8 >> 16))
|
|
if (0x07FF < multi_byte_utf8) and
|
|
((multi_byte_utf8 < 0xD800) or (0xDFFF < multi_byte_utf8)) {
|
|
|
|
args.src.skip_u32_fast!(actual: 3, worst_case: 3)
|
|
if string_length >= 0xFFF8 {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
|
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
|
continued: 1,
|
|
length: string_length + 3)
|
|
string_length = 0
|
|
continue.string_loop_outer
|
|
}
|
|
string_length += 3
|
|
continue.string_loop_inner
|
|
}
|
|
}
|
|
|
|
} else if char == 0x05 { // 4-byte UTF-8.
|
|
if args.src.length() < 4 {
|
|
if string_length > 0 {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
|
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
|
continued: 1,
|
|
length: string_length)
|
|
string_length = 0
|
|
if args.dst.length() <= 0 {
|
|
continue.string_loop_outer
|
|
}
|
|
}
|
|
if args.src.is_closed() {
|
|
if this.quirks[QUIRK_REPLACE_INVALID_UNICODE - QUIRKS_BASE] {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) |
|
|
base.UNICODE__REPLACEMENT_CHARACTER,
|
|
continued: 1,
|
|
length: 1)
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
continue.string_loop_outer
|
|
}
|
|
return "#bad UTF-8"
|
|
}
|
|
yield? base."$short read"
|
|
continue.string_loop_outer
|
|
}
|
|
multi_byte_utf8 = args.src.peek_u32le()
|
|
if (multi_byte_utf8 & 0xC0C0_C000) == 0x8080_8000 {
|
|
multi_byte_utf8 = (0x1C_0000 & (multi_byte_utf8 ~mod<< 18)) |
|
|
(0x03_F000 & (multi_byte_utf8 ~mod<< 4)) |
|
|
(0x00_0FC0 & (multi_byte_utf8 >> 10)) |
|
|
(0x00_003F & (multi_byte_utf8 >> 24))
|
|
if (0xFFFF < multi_byte_utf8) and (multi_byte_utf8 <= 0x10_FFFF) {
|
|
args.src.skip_u32_fast!(actual: 4, worst_case: 4)
|
|
if string_length >= 0xFFF8 {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
|
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
|
continued: 1,
|
|
length: string_length + 4)
|
|
string_length = 0
|
|
continue.string_loop_outer
|
|
}
|
|
string_length += 4
|
|
continue.string_loop_inner
|
|
}
|
|
}
|
|
}
|
|
|
|
if string_length > 0 {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
|
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
|
continued: 1,
|
|
length: string_length)
|
|
string_length = 0
|
|
if args.dst.length() <= 0 {
|
|
continue.string_loop_outer
|
|
}
|
|
}
|
|
if (char & 0x80) <> 0 {
|
|
if this.quirks[QUIRK_ALLOW_ASCII_CONTROL_CODES - QUIRKS_BASE] {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) |
|
|
((char & 0x7F) as base.u32),
|
|
continued: 1,
|
|
length: 1)
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
continue.string_loop_outer
|
|
}
|
|
if char == 0x8A {
|
|
return "#bad new-line in a string"
|
|
}
|
|
return "#bad C0 control code"
|
|
}
|
|
if this.quirks[QUIRK_REPLACE_INVALID_UNICODE - QUIRKS_BASE] {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__UNICODE_CODE_POINT << 21) |
|
|
base.UNICODE__REPLACEMENT_CHARACTER,
|
|
continued: 1,
|
|
length: 1)
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
continue.string_loop_outer
|
|
}
|
|
return "#bad UTF-8"
|
|
} endwhile.string_loop_inner
|
|
} endwhile.string_loop_outer
|
|
|
|
// Emit the trailing '"'.
|
|
while true {
|
|
if args.src.length() <= 0 {
|
|
if args.src.is_closed() {
|
|
return "#bad input"
|
|
}
|
|
yield? base."$short read"
|
|
continue
|
|
}
|
|
if args.dst.length() <= 0 {
|
|
yield? base."$short write"
|
|
continue
|
|
}
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
|
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
|
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
|
base.TOKEN__VBD__STRING__DEFINITELY_ASCII |
|
|
base.TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP,
|
|
continued: 0,
|
|
length: 1)
|
|
break
|
|
} endwhile
|
|
|
|
// As above, expect must have contained EXPECT_STRING. If it didn't
|
|
// also contain EXPECT_NUMBER (excluding EXPECT_COMMENT) then we
|
|
// were parsing an object key and the next token should be ':'.
|
|
if 0 == (expect & ((1 as base.u32) << CLASS_NUMBER)) {
|
|
expect = EXPECT_COLON
|
|
continue.outer
|
|
}
|
|
break.goto_parsed_a_leaf_value
|
|
// -------- END parse strings.
|
|
|
|
} else if class == CLASS_COMMA {
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
// The ',' is punctuation (filler).
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__FILLER << 21) |
|
|
base.TOKEN__VBD__FILLER__PUNCTUATION,
|
|
continued: 0,
|
|
length: 1)
|
|
// What's valid after a comma depends on whether or not we're in an
|
|
// array or an object.
|
|
if 0 == (expect & ((1 as base.u32) << CLASS_CLOSE_SQUARE_BRACKET)) {
|
|
if this.quirks[QUIRK_ALLOW_EXTRA_COMMA - QUIRKS_BASE] {
|
|
expect = EXPECT_STRING | EXPECT_CLOSE_CURLY_BRACE
|
|
} else {
|
|
expect = EXPECT_STRING
|
|
}
|
|
} else {
|
|
if this.quirks[QUIRK_ALLOW_EXTRA_COMMA - QUIRKS_BASE] {
|
|
expect = EXPECT_VALUE | EXPECT_CLOSE_SQUARE_BRACKET
|
|
} else {
|
|
expect = EXPECT_VALUE
|
|
}
|
|
}
|
|
continue.outer
|
|
|
|
} else if class == CLASS_COLON {
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
// The ':' is punctuation (filler).
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__FILLER << 21) |
|
|
base.TOKEN__VBD__FILLER__PUNCTUATION,
|
|
continued: 0,
|
|
length: 1)
|
|
expect = EXPECT_VALUE
|
|
continue.outer
|
|
|
|
} else if class == CLASS_NUMBER {
|
|
// -------- BEGIN parse numbers.
|
|
while true,
|
|
pre args.dst.length() > 0,
|
|
{
|
|
number_length = this.decode_number!(src: args.src)
|
|
number_status = number_length >> 8
|
|
vminor = (base.TOKEN__VBC__NUMBER << 21) |
|
|
base.TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT |
|
|
base.TOKEN__VBD__NUMBER__CONTENT_INTEGER_SIGNED |
|
|
base.TOKEN__VBD__NUMBER__FORMAT_TEXT
|
|
if (number_length & 0x80) <> 0 {
|
|
vminor = (base.TOKEN__VBC__NUMBER << 21) |
|
|
base.TOKEN__VBD__NUMBER__CONTENT_FLOATING_POINT |
|
|
base.TOKEN__VBD__NUMBER__FORMAT_TEXT
|
|
}
|
|
number_length = number_length & 0x7F
|
|
if number_status == 0 {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: vminor,
|
|
continued: 0,
|
|
length: number_length)
|
|
break
|
|
}
|
|
|
|
while number_length > 0 {
|
|
number_length -= 1
|
|
if args.src.can_undo_byte() {
|
|
args.src.undo_byte!()
|
|
} else {
|
|
return "#internal error: inconsistent I/O"
|
|
}
|
|
} endwhile
|
|
|
|
if number_status == 1 {
|
|
if this.quirks[QUIRK_ALLOW_INF_NAN_NUMBERS - QUIRKS_BASE] {
|
|
this.decode_inf_nan?(dst: args.dst, src: args.src)
|
|
break
|
|
}
|
|
return "#bad input"
|
|
} else if number_status == 2 {
|
|
return "#unsupported number length"
|
|
} else {
|
|
yield? base."$short read"
|
|
while args.dst.length() <= 0,
|
|
post args.dst.length() > 0,
|
|
{
|
|
yield? base."$short write"
|
|
} endwhile
|
|
}
|
|
} endwhile
|
|
break.goto_parsed_a_leaf_value
|
|
// -------- END parse numbers.
|
|
|
|
} else if class == CLASS_OPEN_CURLY_BRACE {
|
|
vminor = (base.TOKEN__VBC__STRUCTURE << 21) |
|
|
base.TOKEN__VBD__STRUCTURE__PUSH |
|
|
base.TOKEN__VBD__STRUCTURE__FROM_NONE |
|
|
base.TOKEN__VBD__STRUCTURE__TO_DICT
|
|
if depth == 0 {
|
|
// No-op.
|
|
} else if 0 <> (expect_after_value & ((1 as base.u32) << CLASS_CLOSE_CURLY_BRACE)) {
|
|
vminor = (base.TOKEN__VBC__STRUCTURE << 21) |
|
|
base.TOKEN__VBD__STRUCTURE__PUSH |
|
|
base.TOKEN__VBD__STRUCTURE__FROM_DICT |
|
|
base.TOKEN__VBD__STRUCTURE__TO_DICT
|
|
} else {
|
|
vminor = (base.TOKEN__VBC__STRUCTURE << 21) |
|
|
base.TOKEN__VBD__STRUCTURE__PUSH |
|
|
base.TOKEN__VBD__STRUCTURE__FROM_LIST |
|
|
base.TOKEN__VBD__STRUCTURE__TO_DICT
|
|
}
|
|
if depth >= 1024 {
|
|
return "#unsupported recursion depth"
|
|
}
|
|
stack_byte = depth / 32
|
|
stack_bit = depth & 31
|
|
this.stack[stack_byte] |= (1 as base.u32) << stack_bit
|
|
depth += 1
|
|
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: vminor,
|
|
continued: 0,
|
|
length: 1)
|
|
expect = EXPECT_CLOSE_CURLY_BRACE | EXPECT_STRING
|
|
expect_after_value = EXPECT_CLOSE_CURLY_BRACE | EXPECT_COMMA
|
|
continue.outer
|
|
|
|
} else if class == CLASS_CLOSE_CURLY_BRACE {
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
if depth <= 1 {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__STRUCTURE << 21) |
|
|
base.TOKEN__VBD__STRUCTURE__POP |
|
|
base.TOKEN__VBD__STRUCTURE__FROM_DICT |
|
|
base.TOKEN__VBD__STRUCTURE__TO_NONE,
|
|
continued: 0,
|
|
length: 1)
|
|
break.outer
|
|
}
|
|
depth -= 1
|
|
stack_byte = (depth - 1) / 32
|
|
stack_bit = (depth - 1) & 31
|
|
if 0 == (this.stack[stack_byte] & ((1 as base.u32) << stack_bit)) {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__STRUCTURE << 21) |
|
|
base.TOKEN__VBD__STRUCTURE__POP |
|
|
base.TOKEN__VBD__STRUCTURE__FROM_DICT |
|
|
base.TOKEN__VBD__STRUCTURE__TO_LIST,
|
|
continued: 0,
|
|
length: 1)
|
|
expect = EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA
|
|
expect_after_value = EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA
|
|
} else {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__STRUCTURE << 21) |
|
|
base.TOKEN__VBD__STRUCTURE__POP |
|
|
base.TOKEN__VBD__STRUCTURE__FROM_DICT |
|
|
base.TOKEN__VBD__STRUCTURE__TO_DICT,
|
|
continued: 0,
|
|
length: 1)
|
|
expect = EXPECT_CLOSE_CURLY_BRACE | EXPECT_COMMA
|
|
expect_after_value = EXPECT_CLOSE_CURLY_BRACE | EXPECT_COMMA
|
|
}
|
|
continue.outer
|
|
|
|
} else if class == CLASS_OPEN_SQUARE_BRACKET {
|
|
vminor = (base.TOKEN__VBC__STRUCTURE << 21) |
|
|
base.TOKEN__VBD__STRUCTURE__PUSH |
|
|
base.TOKEN__VBD__STRUCTURE__FROM_NONE |
|
|
base.TOKEN__VBD__STRUCTURE__TO_LIST
|
|
if depth == 0 {
|
|
// No-op.
|
|
} else if 0 <> (expect_after_value & ((1 as base.u32) << CLASS_CLOSE_CURLY_BRACE)) {
|
|
vminor = (base.TOKEN__VBC__STRUCTURE << 21) |
|
|
base.TOKEN__VBD__STRUCTURE__PUSH |
|
|
base.TOKEN__VBD__STRUCTURE__FROM_DICT |
|
|
base.TOKEN__VBD__STRUCTURE__TO_LIST
|
|
} else {
|
|
vminor = (base.TOKEN__VBC__STRUCTURE << 21) |
|
|
base.TOKEN__VBD__STRUCTURE__PUSH |
|
|
base.TOKEN__VBD__STRUCTURE__FROM_LIST |
|
|
base.TOKEN__VBD__STRUCTURE__TO_LIST
|
|
}
|
|
if depth >= 1024 {
|
|
return "#unsupported recursion depth"
|
|
}
|
|
stack_byte = depth / 32
|
|
stack_bit = depth & 31
|
|
this.stack[stack_byte] &= 0xFFFF_FFFF ^ ((1 as base.u32) << stack_bit)
|
|
depth += 1
|
|
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: vminor,
|
|
continued: 0,
|
|
length: 1)
|
|
expect = EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_VALUE
|
|
expect_after_value = EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA
|
|
continue.outer
|
|
|
|
} else if class == CLASS_CLOSE_SQUARE_BRACKET {
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
if depth <= 1 {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__STRUCTURE << 21) |
|
|
base.TOKEN__VBD__STRUCTURE__POP |
|
|
base.TOKEN__VBD__STRUCTURE__FROM_LIST |
|
|
base.TOKEN__VBD__STRUCTURE__TO_NONE,
|
|
continued: 0,
|
|
length: 1)
|
|
break.outer
|
|
}
|
|
depth -= 1
|
|
stack_byte = (depth - 1) / 32
|
|
stack_bit = (depth - 1) & 31
|
|
if 0 == (this.stack[stack_byte] & ((1 as base.u32) << stack_bit)) {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__STRUCTURE << 21) |
|
|
base.TOKEN__VBD__STRUCTURE__POP |
|
|
base.TOKEN__VBD__STRUCTURE__FROM_LIST |
|
|
base.TOKEN__VBD__STRUCTURE__TO_LIST,
|
|
continued: 0,
|
|
length: 1)
|
|
expect = EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA
|
|
expect_after_value = EXPECT_CLOSE_SQUARE_BRACKET | EXPECT_COMMA
|
|
} else {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__STRUCTURE << 21) |
|
|
base.TOKEN__VBD__STRUCTURE__POP |
|
|
base.TOKEN__VBD__STRUCTURE__FROM_LIST |
|
|
base.TOKEN__VBD__STRUCTURE__TO_DICT,
|
|
continued: 0,
|
|
length: 1)
|
|
expect = EXPECT_CLOSE_CURLY_BRACE | EXPECT_COMMA
|
|
expect_after_value = EXPECT_CLOSE_CURLY_BRACE | EXPECT_COMMA
|
|
}
|
|
continue.outer
|
|
|
|
} else if class == CLASS_FALSE {
|
|
match = args.src.match7(a: '\x05false'le)
|
|
if match == 0 {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__LITERAL << 21) |
|
|
base.TOKEN__VBD__LITERAL__FALSE,
|
|
continued: 0,
|
|
length: 5)
|
|
if args.src.length() < 5 {
|
|
return "#internal error: inconsistent I/O"
|
|
}
|
|
args.src.skip_u32_fast!(actual: 5, worst_case: 5)
|
|
break.goto_parsed_a_leaf_value
|
|
} else if match == 1 {
|
|
yield? base."$short read"
|
|
continue.outer
|
|
}
|
|
|
|
} else if class == CLASS_TRUE {
|
|
match = args.src.match7(a: '\x04true'le)
|
|
if match == 0 {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__LITERAL << 21) |
|
|
base.TOKEN__VBD__LITERAL__TRUE,
|
|
continued: 0,
|
|
length: 4)
|
|
if args.src.length() < 4 {
|
|
return "#internal error: inconsistent I/O"
|
|
}
|
|
args.src.skip_u32_fast!(actual: 4, worst_case: 4)
|
|
break.goto_parsed_a_leaf_value
|
|
} else if match == 1 {
|
|
yield? base."$short read"
|
|
continue.outer
|
|
}
|
|
|
|
} else if class == CLASS_NULL_NAN_INF {
|
|
match = args.src.match7(a: '\x04null'le)
|
|
if match == 0 {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__LITERAL << 21) |
|
|
base.TOKEN__VBD__LITERAL__NULL,
|
|
continued: 0,
|
|
length: 4)
|
|
if args.src.length() < 4 {
|
|
return "#internal error: inconsistent I/O"
|
|
}
|
|
args.src.skip_u32_fast!(actual: 4, worst_case: 4)
|
|
break.goto_parsed_a_leaf_value
|
|
} else if match == 1 {
|
|
yield? base."$short read"
|
|
continue.outer
|
|
}
|
|
|
|
if this.quirks[QUIRK_ALLOW_INF_NAN_NUMBERS - QUIRKS_BASE] {
|
|
this.decode_inf_nan?(dst: args.dst, src: args.src)
|
|
break.goto_parsed_a_leaf_value
|
|
}
|
|
|
|
} else if class == CLASS_COMMENT {
|
|
if this.quirks[QUIRK_ALLOW_COMMENT_BLOCK - QUIRKS_BASE] or
|
|
this.quirks[QUIRK_ALLOW_COMMENT_LINE - QUIRKS_BASE] {
|
|
this.decode_comment?(dst: args.dst, src: args.src)
|
|
if this.comment_type > 0 {
|
|
continue.outer
|
|
}
|
|
}
|
|
}
|
|
|
|
return "#bad input"
|
|
}} endwhile.goto_parsed_a_leaf_value
|
|
|
|
// We've just parsed a leaf (non-container) value: literal (null,
|
|
// false, true), number or string.
|
|
if depth == 0 {
|
|
break.outer
|
|
}
|
|
expect = expect_after_value
|
|
} endwhile.outer
|
|
|
|
if this.quirks[QUIRK_ALLOW_TRAILING_FILLER - QUIRKS_BASE] or
|
|
this.quirks[QUIRK_EXPECT_TRAILING_NEW_LINE_OR_EOF - QUIRKS_BASE] {
|
|
this.decode_trailer?(dst: args.dst, src: args.src)
|
|
}
|
|
|
|
this.end_of_data = true
|
|
}
|
|
|
|
pri func decoder.decode_number!(src: base.io_reader) base.u32[..= 0x3FF] {
|
|
var c : base.u8
|
|
var n : base.u32[..= 0x3FF]
|
|
var floating_point : base.u32[..= 0x80]
|
|
|
|
while.goto_done true {{
|
|
n = 0
|
|
|
|
// Peek.
|
|
if args.src.length() <= 0 {
|
|
if not args.src.is_closed() {
|
|
n |= 0x300
|
|
}
|
|
break.goto_done
|
|
}
|
|
c = args.src.peek_u8()
|
|
|
|
// Scan the optional minus sign.
|
|
if c <> '-' {
|
|
assert args.src.length() > 0
|
|
assert n <= 1
|
|
} else {
|
|
n += 1
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
|
|
// Peek.
|
|
if args.src.length() <= 0 {
|
|
if not args.src.is_closed() {
|
|
n |= 0x300
|
|
}
|
|
n |= 0x100 // A '-' without digits is invalid.
|
|
break.goto_done
|
|
}
|
|
c = args.src.peek_u8()
|
|
|
|
assert args.src.length() > 0
|
|
assert n <= 1
|
|
}
|
|
|
|
// Scan the opening digits.
|
|
if c == '0' {
|
|
n += 1
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
assert n <= 99
|
|
} else {
|
|
n = this.decode_digits!(src: args.src, n: n)
|
|
if n > 99 {
|
|
break.goto_done
|
|
}
|
|
assert n <= 99
|
|
}
|
|
|
|
// Peek.
|
|
if args.src.length() <= 0 {
|
|
if not args.src.is_closed() {
|
|
n |= 0x300
|
|
}
|
|
break.goto_done
|
|
}
|
|
c = args.src.peek_u8()
|
|
|
|
// Scan the optional fraction.
|
|
if c <> '.' {
|
|
assert args.src.length() > 0
|
|
assert n <= 99
|
|
} else {
|
|
if n >= 99 {
|
|
n |= 0x200
|
|
break.goto_done
|
|
}
|
|
n += 1
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
floating_point = 0x80
|
|
|
|
n = this.decode_digits!(src: args.src, n: n)
|
|
if n > 99 {
|
|
break.goto_done
|
|
}
|
|
|
|
// Peek.
|
|
if args.src.length() <= 0 {
|
|
if not args.src.is_closed() {
|
|
n |= 0x300
|
|
}
|
|
break.goto_done
|
|
}
|
|
c = args.src.peek_u8()
|
|
|
|
assert args.src.length() > 0
|
|
assert n <= 99
|
|
}
|
|
|
|
// Scan the optional 'E' or 'e'.
|
|
if (c <> 'E') and (c <> 'e') {
|
|
break.goto_done
|
|
}
|
|
if n >= 99 {
|
|
n |= 0x200
|
|
break.goto_done
|
|
}
|
|
n += 1
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
floating_point = 0x80
|
|
assert n <= 99
|
|
|
|
// Peek.
|
|
if args.src.length() <= 0 {
|
|
if not args.src.is_closed() {
|
|
n |= 0x300
|
|
}
|
|
n |= 0x100 // An 'E' or 'e' without digits is invalid.
|
|
break.goto_done
|
|
}
|
|
c = args.src.peek_u8()
|
|
|
|
// Scan the optional '+' or '-'.
|
|
if (c <> '+') and (c <> '-') {
|
|
assert n <= 99
|
|
} else {
|
|
if n >= 99 {
|
|
n |= 0x200
|
|
break.goto_done
|
|
}
|
|
n += 1
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
assert n <= 99
|
|
}
|
|
|
|
// Scan the exponent digits.
|
|
n = this.decode_digits!(src: args.src, n: n)
|
|
|
|
break.goto_done
|
|
}} endwhile.goto_done
|
|
|
|
return n | floating_point
|
|
}
|
|
|
|
pri func decoder.decode_digits!(src: base.io_reader, n: base.u32[..= 99]) base.u32[..= 0x3FF] {
|
|
var c : base.u8
|
|
var n : base.u32[..= 0x3FF]
|
|
|
|
n = args.n
|
|
while true {
|
|
if args.src.length() <= 0 {
|
|
if not args.src.is_closed() {
|
|
n |= 0x300
|
|
}
|
|
break
|
|
}
|
|
c = args.src.peek_u8()
|
|
if 0x00 == LUT_DECIMAL_DIGITS[c] {
|
|
break
|
|
}
|
|
// Cap DECODER_NUMBER_LENGTH_MAX_INCL at an arbitrary value, 99. The
|
|
// caller's src.data.len should therefore be at least 100, also known
|
|
// as DECODER_SRC_IO_BUFFER_LENGTH_MIN_INCL.
|
|
//
|
|
// An example of a JSON number that is 81 bytes long is:
|
|
// https://github.com/nst/JSONTestSuite/blob/master/test_parsing/y_number_double_close_to_zero.json
|
|
//
|
|
// Note that 99 (in hex, 0x63) is less than 0x80, so we can use 0x80 as
|
|
// a flag bit in func decoder.decode_number.
|
|
if n >= 99 {
|
|
n |= 0x200
|
|
break
|
|
}
|
|
n += 1
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
} endwhile
|
|
if n == args.n {
|
|
n |= 0x100
|
|
}
|
|
return n
|
|
}
|
|
|
|
pri func decoder.decode_leading?(dst: base.token_writer, src: base.io_reader) {
|
|
var c : base.u8
|
|
var u : base.u32
|
|
|
|
this.allow_leading_ars =
|
|
this.quirks[QUIRK_ALLOW_LEADING_ASCII_RECORD_SEPARATOR - QUIRKS_BASE]
|
|
this.allow_leading_ubom =
|
|
this.quirks[QUIRK_ALLOW_LEADING_UNICODE_BYTE_ORDER_MARK - QUIRKS_BASE]
|
|
|
|
while this.allow_leading_ars or this.allow_leading_ubom {
|
|
if args.dst.length() <= 0 {
|
|
yield? base."$short write"
|
|
continue
|
|
}
|
|
if args.src.length() <= 0 {
|
|
if args.src.is_closed() {
|
|
break
|
|
}
|
|
yield? base."$short read"
|
|
continue
|
|
}
|
|
c = args.src.peek_u8()
|
|
if (c == 0x1E) and this.allow_leading_ars {
|
|
this.allow_leading_ars = false
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0, value_minor: 0, continued: 0, length: 1)
|
|
continue
|
|
} else if (c == 0xEF) and this.allow_leading_ubom {
|
|
if args.src.length() < 3 {
|
|
if args.src.is_closed() {
|
|
break
|
|
}
|
|
yield? base."$short read"
|
|
continue
|
|
}
|
|
u = args.src.peek_u24le_as_u32()
|
|
if u == 0xBF_BBEF {
|
|
this.allow_leading_ubom = false
|
|
args.src.skip_u32_fast!(actual: 3, worst_case: 3)
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0, value_minor: 0, continued: 0, length: 3)
|
|
continue
|
|
}
|
|
}
|
|
break
|
|
} endwhile
|
|
}
|
|
|
|
pri func decoder.decode_comment?(dst: base.token_writer, src: base.io_reader) {
|
|
var c : base.u8
|
|
var c2 : base.u16
|
|
var length : base.u32[..= 0xFFFD]
|
|
|
|
this.comment_type = 0
|
|
|
|
while (args.dst.length() <= 0) or (args.src.length() <= 1),
|
|
post args.dst.length() > 0,
|
|
post args.src.length() > 1,
|
|
{
|
|
if args.dst.length() <= 0 {
|
|
yield? base."$short write"
|
|
continue
|
|
}
|
|
if args.src.is_closed() {
|
|
return ok
|
|
}
|
|
yield? base."$short read"
|
|
} endwhile
|
|
c2 = args.src.peek_u16le()
|
|
|
|
if (c2 == '/*'le) and this.quirks[QUIRK_ALLOW_COMMENT_BLOCK - QUIRKS_BASE] {
|
|
args.src.skip_u32_fast!(actual: 2, worst_case: 2)
|
|
length = 2
|
|
|
|
while.comment_block true,
|
|
pre args.dst.length() > 0,
|
|
{
|
|
if args.src.length() <= 1 {
|
|
if length > 0 {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__FILLER << 21) |
|
|
base.TOKEN__VBD__FILLER__COMMENT_BLOCK,
|
|
continued: 1,
|
|
length: length)
|
|
}
|
|
if args.src.is_closed() {
|
|
return "#bad input"
|
|
}
|
|
yield? base."$short read"
|
|
while args.dst.length() <= 0,
|
|
post args.dst.length() > 0,
|
|
{
|
|
yield? base."$short write"
|
|
} endwhile
|
|
length = 0
|
|
continue.comment_block
|
|
}
|
|
|
|
c2 = args.src.peek_u16le()
|
|
if c2 == '*/'le {
|
|
args.src.skip_u32_fast!(actual: 2, worst_case: 2)
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__FILLER << 21) |
|
|
base.TOKEN__VBD__FILLER__COMMENT_BLOCK,
|
|
continued: 0,
|
|
length: length + 2)
|
|
this.comment_type = 1
|
|
return ok
|
|
}
|
|
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
if length >= 0xFFFD {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__FILLER << 21) |
|
|
base.TOKEN__VBD__FILLER__COMMENT_BLOCK,
|
|
continued: 1,
|
|
length: length + 1)
|
|
while args.dst.length() <= 0,
|
|
post args.dst.length() > 0,
|
|
{
|
|
yield? base."$short write"
|
|
} endwhile
|
|
length = 0
|
|
continue.comment_block
|
|
}
|
|
length += 1
|
|
} endwhile.comment_block
|
|
|
|
} else if (c2 == '//'le) and this.quirks[QUIRK_ALLOW_COMMENT_LINE - QUIRKS_BASE] {
|
|
args.src.skip_u32_fast!(actual: 2, worst_case: 2)
|
|
length = 2
|
|
|
|
while.comment_line true,
|
|
pre args.dst.length() > 0,
|
|
{
|
|
if args.src.length() <= 0 {
|
|
if args.src.is_closed() {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__FILLER << 21) |
|
|
base.TOKEN__VBD__FILLER__COMMENT_LINE,
|
|
continued: 0,
|
|
length: length)
|
|
this.comment_type = 2
|
|
return ok
|
|
} else if length > 0 {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__FILLER << 21) |
|
|
base.TOKEN__VBD__FILLER__COMMENT_LINE,
|
|
continued: 1,
|
|
length: length)
|
|
}
|
|
yield? base."$short read"
|
|
while args.dst.length() <= 0,
|
|
post args.dst.length() > 0,
|
|
{
|
|
yield? base."$short write"
|
|
} endwhile
|
|
length = 0
|
|
continue.comment_line
|
|
}
|
|
|
|
c = args.src.peek_u8()
|
|
if c == '\n' {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__FILLER << 21) |
|
|
base.TOKEN__VBD__FILLER__COMMENT_LINE,
|
|
continued: 0,
|
|
length: length)
|
|
this.comment_type = 2
|
|
return ok
|
|
}
|
|
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
if length >= 0xFFFD {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__FILLER << 21) |
|
|
base.TOKEN__VBD__FILLER__COMMENT_LINE,
|
|
continued: 1,
|
|
length: length + 1)
|
|
while args.dst.length() <= 0,
|
|
post args.dst.length() > 0,
|
|
{
|
|
yield? base."$short write"
|
|
} endwhile
|
|
length = 0
|
|
continue.comment_line
|
|
}
|
|
length += 1
|
|
} endwhile.comment_line
|
|
}
|
|
}
|
|
|
|
pri func decoder.decode_inf_nan?(dst: base.token_writer, src: base.io_reader) {
|
|
var c4 : base.u32
|
|
var neg : base.u32[..= 1]
|
|
|
|
while true {
|
|
if args.dst.length() <= 0 {
|
|
yield? base."$short write"
|
|
continue
|
|
}
|
|
if args.src.length() <= 2 {
|
|
if args.src.is_closed() {
|
|
return "#bad input"
|
|
}
|
|
yield? base."$short read"
|
|
continue
|
|
}
|
|
|
|
// Bitwise or'ing with 0x20 converts upper case ASCII to lower case.
|
|
|
|
c4 = args.src.peek_u24le_as_u32()
|
|
if (c4 | 0x20_2020) == 'inf'le {
|
|
if args.src.length() > 7 {
|
|
if (args.src.peek_u64le() | 0x2020_2020_2020_2020) == 'infinity'le {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__NUMBER << 21) |
|
|
base.TOKEN__VBD__NUMBER__CONTENT_POS_INF,
|
|
continued: 0,
|
|
length: 8)
|
|
args.src.skip_u32_fast!(actual: 8, worst_case: 8)
|
|
return ok
|
|
}
|
|
} else if not args.src.is_closed() {
|
|
yield? base."$short read"
|
|
continue
|
|
}
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__NUMBER << 21) |
|
|
base.TOKEN__VBD__NUMBER__CONTENT_POS_INF,
|
|
continued: 0,
|
|
length: 3)
|
|
args.src.skip_u32_fast!(actual: 3, worst_case: 3)
|
|
return ok
|
|
|
|
} else if (c4 | 0x20_2020) == 'nan'le {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__NUMBER << 21) |
|
|
base.TOKEN__VBD__NUMBER__CONTENT_POS_NAN,
|
|
continued: 0,
|
|
length: 3)
|
|
args.src.skip_u32_fast!(actual: 3, worst_case: 3)
|
|
return ok
|
|
} else if (c4 & 0xFF) == '+' {
|
|
neg = 0
|
|
} else if (c4 & 0xFF) == '-' {
|
|
neg = 1
|
|
} else {
|
|
return "#bad input"
|
|
}
|
|
|
|
if args.src.length() <= 3 {
|
|
if args.src.is_closed() {
|
|
return "#bad input"
|
|
}
|
|
yield? base."$short read"
|
|
continue
|
|
}
|
|
|
|
c4 = args.src.peek_u32le() >> 8
|
|
if (c4 | 0x20_2020) == 'inf'le {
|
|
if args.src.length() > 8 {
|
|
if (args.src.peek_u64le_at(offset: 1) | 0x2020_2020_2020_2020) == 'infinity'le {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__NUMBER << 21) |
|
|
(base.TOKEN__VBD__NUMBER__CONTENT_POS_INF >> neg),
|
|
continued: 0,
|
|
length: 9)
|
|
args.src.skip_u32_fast!(actual: 9, worst_case: 9)
|
|
return ok
|
|
}
|
|
} else if not args.src.is_closed() {
|
|
yield? base."$short read"
|
|
continue
|
|
}
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__NUMBER << 21) |
|
|
(base.TOKEN__VBD__NUMBER__CONTENT_POS_INF >> neg),
|
|
continued: 0,
|
|
length: 4)
|
|
args.src.skip_u32_fast!(actual: 4, worst_case: 4)
|
|
return ok
|
|
|
|
} else if (c4 | 0x20_2020) == 'nan'le {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0,
|
|
value_minor: (base.TOKEN__VBC__NUMBER << 21) |
|
|
(base.TOKEN__VBD__NUMBER__CONTENT_POS_NAN >> neg),
|
|
continued: 0,
|
|
length: 4)
|
|
args.src.skip_u32_fast!(actual: 4, worst_case: 4)
|
|
return ok
|
|
}
|
|
|
|
return "#bad input"
|
|
} endwhile
|
|
}
|
|
|
|
pri func decoder.decode_trailer?(dst: base.token_writer, src: base.io_reader) {
|
|
var c : base.u8
|
|
var whitespace_length : base.u32[..= 0xFFFE]
|
|
|
|
if this.quirks[QUIRK_EXPECT_TRAILING_NEW_LINE_OR_EOF - QUIRKS_BASE] {
|
|
this.trailer_stop = '\n'
|
|
} else {
|
|
this.trailer_stop = 0
|
|
}
|
|
|
|
while.outer true {
|
|
if args.dst.length() <= 0 {
|
|
yield? base."$short write"
|
|
continue.outer
|
|
}
|
|
|
|
whitespace_length = 0
|
|
while.inner true,
|
|
pre args.dst.length() > 0,
|
|
{
|
|
if args.src.length() <= 0 {
|
|
if whitespace_length > 0 {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0, value_minor: 0, continued: 0, length: whitespace_length)
|
|
}
|
|
if args.src.is_closed() {
|
|
break.outer
|
|
}
|
|
yield? base."$short read"
|
|
continue.outer
|
|
}
|
|
|
|
c = args.src.peek_u8()
|
|
if LUT_CLASSES[c] <> CLASS_WHITESPACE {
|
|
if whitespace_length > 0 {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0, value_minor: 0, continued: 0, length: whitespace_length)
|
|
}
|
|
if this.trailer_stop > 0 {
|
|
return "#bad input"
|
|
}
|
|
this.decode_comment?(dst: args.dst, src: args.src)
|
|
if this.comment_type > 0 {
|
|
continue.outer
|
|
}
|
|
return ok
|
|
}
|
|
|
|
args.src.skip_u32_fast!(actual: 1, worst_case: 1)
|
|
if (whitespace_length >= 0xFFFE) or (c == this.trailer_stop) {
|
|
args.dst.write_simple_token_fast!(
|
|
value_major: 0, value_minor: 0, continued: 0, length: whitespace_length + 1)
|
|
if c == this.trailer_stop {
|
|
return ok
|
|
}
|
|
continue.outer
|
|
}
|
|
whitespace_length += 1
|
|
} endwhile.inner
|
|
} endwhile.outer
|
|
}
|