mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-19 16:47:28 +01:00
Add TOKEN__VBD__STRING__CHAIN_ETC
This commit is contained in:
1 parent
0eee231ce6
commit
b9afc26387
10 files changed
+117
-57
No files matched your search
@@ -206,6 +206,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_ASCII |
|
||||
base.TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP,
|
||||
continued: 1,
|
||||
@@ -228,6 +229,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: string_length)
|
||||
@@ -260,6 +262,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: string_length + 4)
|
||||
@@ -279,6 +282,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: 0xFFFC)
|
||||
@@ -294,6 +298,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: string_length)
|
||||
@@ -307,6 +312,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: string_length)
|
||||
@@ -594,6 +600,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
args.dst.write_simple_token_fast!(
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_4_SRC_BACKSLASH_X,
|
||||
continued: 1,
|
||||
length: backslash_x_length)
|
||||
@@ -610,6 +618,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: string_length)
|
||||
@@ -646,6 +655,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: string_length + 2)
|
||||
@@ -663,6 +673,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: string_length)
|
||||
@@ -703,6 +714,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: string_length + 3)
|
||||
@@ -721,6 +733,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: string_length)
|
||||
@@ -760,6 +773,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: string_length + 4)
|
||||
@@ -777,6 +791,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CONVERT_1_DST_1_SRC_COPY,
|
||||
continued: 1,
|
||||
length: string_length)
|
||||
@@ -830,6 +845,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader, wor
|
||||
value_major: 0,
|
||||
value_minor: (base.TOKEN__VBC__STRING << 21) |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8 |
|
||||
base.TOKEN__VBD__STRING__DEFINITELY_ASCII |
|
||||
base.TOKEN__VBD__STRING__CONVERT_0_DST_1_SRC_DROP,
|
||||
continued: 0,
|
||||
|
||||
@@ -81,17 +81,24 @@ pub const QUIRK_ALLOW_BACKSLASH_V : base.u32 = 0x4909_9400 | 0x07
|
||||
// "\x", "\x9", "\x9$" and "\X99" are all still rejected.
|
||||
//
|
||||
// It is up to the caller how to interpret the two hexadecimal digits. There
|
||||
// are two sensible choices: as a byte or as a Unicode code point.
|
||||
// are two sensible choices: as-a-byte or as-a-Unicode-code-point.
|
||||
//
|
||||
// As a byte means decoding to 5 bytes: 0x61, 0x62, 0x63, 0xEF and 0x7A.
|
||||
// As-a-byte means decoding to 5 bytes: 0x61, 0x62, 0x63, 0xEF and 0x7A.
|
||||
// Decoded strings are byte strings, no longer guaranteed to be valid UTF-8 and
|
||||
// even if the overall byte string is valid UTF-8, tokens are also no longer
|
||||
// guaranteed to split on UTF-8 boundaries.
|
||||
//
|
||||
// As a Unicode code point means decoding to 6 bytes: 0x61, 0x62, 0x63, 0xC3,
|
||||
// As-a-Unicode-code-point means decoding to 6 bytes: 0x61, 0x62, 0x63, 0xC3,
|
||||
// 0xAF and 0x7A. The UTF-8 encoding of U+00EF LATIN SMALL LETTER I WITH
|
||||
// DIAERESIS is the two byte sequence (0xC3, 0xAF). Decoded strings are still
|
||||
// valid UTF-8 and tokens still split on UTF-8 boundaries.
|
||||
//
|
||||
// The parser cannot know which interpretation the caller will choose, so it
|
||||
// will still mark its string tokens with TOKEN__VBD__STRING__DEFINITELY_UTF_8
|
||||
// and TOKEN__VBD__STRING__CHAIN_MUST_BE_UTF_8, like all other JSON string
|
||||
// tokens, even though the as-a-byte interpretation can result in invalid
|
||||
// UTF-8. If the caller chooses as-a-byte, it is up to the caller whether and
|
||||
// how to run UTF-8 validation.
|
||||
pub const QUIRK_ALLOW_BACKSLASH_X : base.u32 = 0x4909_9400 | 0x08
|
||||
|
||||
// When this quirk is enabled, e.g. "abc\0z" is accepted as a JSON string,
|
||||
|
||||
Reference in new issue
Block a user