Have std/json reject some invalid UTF-8

This commit is contained in:
Nigel Tao committed 2020-02-26 14:36:23 +11:00
1 parent 9f0eb266e8
commit 3b17d5b28a
4 files changed
+303 -79

No files matched your search

+98 -43
View File
@@ -5290,6 +5290,9 @@ extern "C" {
// ---------------- Status Codes
extern const char* wuffs_json__error__bad_c0_control_code;
extern const char* wuffs_json__error__bad_utf_8;
extern const char* wuffs_json__error__bad_backslash_escape;
extern const char* wuffs_json__error__bad_input;
extern const char* wuffs_json__error__unsupported_number_length;
extern const char* wuffs_json__error__unsupported_recursion_depth;
@@ -18717,6 +18720,11 @@ exit:
// ---------------- Status Codes Implementations
const char* wuffs_json__error__bad_c0_control_code =
"#json: bad C0 control code";
const char* wuffs_json__error__bad_utf_8 = "#json: bad UTF-8";
const char* wuffs_json__error__bad_backslash_escape =
"#json: bad backslash-escape";
const char* wuffs_json__error__bad_input = "#json: bad input";
const char* wuffs_json__error__unsupported_number_length =
"#json: unsupported number length";
@@ -18746,6 +18754,30 @@ static const uint8_t //
0, 0, 0, 0, 0, 0, 0, 0, 0,
};
static const uint8_t //
wuffs_json__lut_chars[256] //
WUFFS_BASE__POTENTIALLY_UNUSED = {
128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 129, 129, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
4, 4, 5, 5, 5, 5, 5, 129, 129, 129, 129, 129, 129, 129,
129, 129, 129, 129,
};
static const uint8_t //
wuffs_json__lut_classes[256] //
WUFFS_BASE__POTENTIALLY_UNUSED = {
@@ -18905,8 +18937,9 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
uint32_t v_stack_bit = 0;
uint32_t v_match = 0;
uint8_t v_c = 0;
uint8_t v_class = 0;
uint8_t v_backslash = 0;
uint8_t v_char = 0;
uint8_t v_class = 0;
uint8_t v_uni4_ok = 0;
uint64_t v_uni4_string = 0;
uint32_t v_uni4_value = 0;
@@ -19051,6 +19084,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
goto label__string_loop__continue;
}
v_string_length = 0;
label__1__continue:;
while (true) {
if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) {
if (v_string_length > 0) {
@@ -19073,29 +19107,28 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
goto label__string_loop__continue;
}
v_c = wuffs_base__load_u8be(iop_a_src);
if (v_c <= 34) {
if (v_c == 34) {
v_char = wuffs_json__lut_chars[v_c];
if (v_char == 0) {
(iop_a_src += 1, wuffs_base__make_empty_struct());
if (v_string_length >= 65534) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(2097696))
(((uint64_t)(2097697))
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(((uint64_t)(v_string_length))))
(((uint64_t)(65535))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
goto label__string_loop__break;
} else if (v_c < 32) {
if (v_string_length > 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(2097697))
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(((uint64_t)(v_string_length))))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
}
status =
wuffs_base__make_status(wuffs_json__error__bad_input);
goto exit;
goto label__string_loop__continue;
}
} else if (v_c == 92) {
v_string_length += 1;
goto label__1__continue;
} else if (v_char == 1) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(2097696)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(((uint64_t)(v_string_length))))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
goto label__string_loop__break;
} else if (v_char == 2) {
if (v_string_length > 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(2097697))
@@ -19109,37 +19142,39 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
}
if (((uint64_t)(io2_a_src - iop_a_src)) < 2) {
if (a_src && a_src->meta.closed) {
status =
wuffs_base__make_status(wuffs_json__error__bad_input);
status = wuffs_base__make_status(
wuffs_json__error__bad_backslash_escape);
goto exit;
}
status = wuffs_base__make_status(
wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(7);
v_string_length = 0;
v_char = 0;
goto label__string_loop__continue;
}
v_backslash =
((uint8_t)((wuffs_base__load_u16le(iop_a_src) >> 8)));
v_c = wuffs_json__lut_backslashes[v_backslash];
if (v_c > 0) {
v_c = ((uint8_t)((wuffs_base__load_u16le(iop_a_src) >> 8)));
v_backslash = wuffs_json__lut_backslashes[v_c];
if (v_backslash > 0) {
(iop_a_src += 2, wuffs_base__make_empty_struct());
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)((4194304 | ((uint64_t)((v_c & 127))))))
(((uint64_t)(
(4194304 | ((uint64_t)((v_backslash & 127))))))
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(2)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__string_loop__continue;
} else if (v_backslash == 117) {
} else if (v_c == 117) {
if (((uint64_t)(io2_a_src - iop_a_src)) < 6) {
if (a_src && a_src->meta.closed) {
status =
wuffs_base__make_status(wuffs_json__error__bad_input);
status = wuffs_base__make_status(
wuffs_json__error__bad_backslash_escape);
goto exit;
}
status = wuffs_base__make_status(
wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(8);
v_string_length = 0;
v_char = 0;
goto label__string_loop__continue;
}
v_uni4_string =
@@ -19175,7 +19210,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
if (((uint64_t)(io2_a_src - iop_a_src)) < 12) {
if (a_src && a_src->meta.closed) {
status = wuffs_base__make_status(
wuffs_json__error__bad_input);
wuffs_json__error__bad_backslash_escape);
goto exit;
}
status = wuffs_base__make_status(
@@ -19183,6 +19218,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(9);
v_string_length = 0;
v_uni4_value = 0;
v_char = 0;
goto label__string_loop__continue;
}
(iop_a_src += 4, wuffs_base__make_empty_struct());
@@ -19242,22 +19278,41 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
}
}
}
status = wuffs_base__make_status(wuffs_json__error__bad_input);
status = wuffs_base__make_status(
wuffs_json__error__bad_backslash_escape);
goto exit;
} else if (v_char <= 16) {
(iop_a_src += 1, wuffs_base__make_empty_struct());
if (v_string_length >= 65534) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(2097697))
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(65535))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
goto label__string_loop__continue;
}
v_string_length += 1;
goto label__1__continue;
}
(iop_a_src += 1, wuffs_base__make_empty_struct());
if (v_string_length >= 65534) {
if (v_string_length > 0) {
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(2097697)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(65535)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
(((uint64_t)(((uint64_t)(v_string_length))))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
v_string_length = 0;
goto label__string_loop__continue;
}
v_string_length += 1;
if (v_char == 128) {
status = wuffs_base__make_status(
wuffs_json__error__bad_c0_control_code);
goto exit;
}
status = wuffs_base__make_status(wuffs_json__error__bad_utf_8);
goto exit;
}
}
label__string_loop__break:;
label__1__continue:;
label__2__continue:;
while (true) {
if (((uint64_t)(io2_a_src - iop_a_src)) <= 0) {
if (a_src && a_src->meta.closed) {
@@ -19267,21 +19322,21 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
status =
wuffs_base__make_status(wuffs_base__suspension__short_read);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(10);
goto label__1__continue;
goto label__2__continue;
}
if (((uint64_t)(io2_a_dst - iop_a_dst)) <= 0) {
status =
wuffs_base__make_status(wuffs_base__suspension__short_write);
WUFFS_BASE__COROUTINE_SUSPENSION_POINT_MAYBE_SUSPEND(11);
goto label__1__continue;
goto label__2__continue;
}
(iop_a_src += 1, wuffs_base__make_empty_struct());
*iop_a_dst++ = wuffs_base__make_token(
(((uint64_t)(0)) << WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(1)) << WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__1__break;
goto label__2__break;
}
label__1__break:;
label__2__break:;
if (0 == (v_expect & 16)) {
v_expect = 8;
goto label__outer__continue;
@@ -19326,7 +19381,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
<< WUFFS_BASE__TOKEN__VALUE__SHIFT) |
(((uint64_t)(((uint64_t)(v_number_length))))
<< WUFFS_BASE__TOKEN__LENGTH__SHIFT));
goto label__2__break;
goto label__3__break;
}
while (v_number_length > 0) {
v_number_length -= 1;
@@ -19356,7 +19411,7 @@ wuffs_json__decoder__decode_tokens(wuffs_json__decoder* self,
}
}
}
label__2__break:;
label__3__break:;
goto label__goto_parsed_a_leaf_value__break;
} else if (v_class == 5) {
v_token_value = 8405009;
+65
View File
@@ -12,6 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub status "#bad C0 control code"
pub status "#bad UTF-8"
pub status "#bad backslash-escape"
pub status "#bad input"
pub status "#unsupported number length"
pub status "#unsupported recursion depth"
@@ -62,6 +65,68 @@ pri const lut_backslashes array[256] base.u8 = [
// 8 9 A B C D E F
]
// lut_chars helps decode bytes within a string:
// - 0x00 is 1-byte UTF-8 (ASCII) but not '"', '\\' or a C0 control code.
// - 0x01 is '"'.
// - 0x02 is '\\'.
// - 0x03 is the start of 2-byte UTF-8.
// - 0x04 is the start of 3-byte UTF-8.
// - 0x05 is the start of 4-byte UTF-8.
// - 0x10 is a UTF-8 tail byte.
// - 0x80 is invalid JSON (C0 control codes).
// - 0x81 is invalid UTF-8.
//
// RFC 3629 (UTF-8) gives this grammar for valid UTF-8:
// UTF8-1 = %x00-7F
// UTF8-2 = %xC2-DF UTF8-tail
// UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) /
// %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail )
// UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) /
// %xF4 %x80-8F 2( UTF8-tail )
// UTF8-tail = %x80-BF
pri const lut_chars array[256] base.u8 = [
// 0 1 2 3 4 5 6 7
// 8 9 A B C D E F
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x00 ..= 0x07. C0 control codes.
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x08 ..= 0x0F. C0 control codes.
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x10 ..= 0x17. C0 control codes.
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0x18 ..= 0x1F. C0 control codes.
0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20 ..= 0x27. UTF-8-1; '"'.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x28 ..= 0x2F. UTF-8-1.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x30 ..= 0x37. UTF-8-1.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x38 ..= 0x3F. UTF-8-1.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x40 ..= 0x47. UTF-8-1.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x48 ..= 0x4F. UTF-8-1.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x50 ..= 0x57. UTF-8-1.
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, // 0x58 ..= 0x5F. UTF-8-1; '\\'.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x60 ..= 0x67. UTF-8-1.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x68 ..= 0x6F. UTF-8-1.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x70 ..= 0x77. UTF-8-1.
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x78 ..= 0x7F. UTF-8-1.
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0x80 ..= 0x87. UTF-8 tail.
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0x88 ..= 0x8F. UTF-8 tail.
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0x90 ..= 0x97. UTF-8 tail.
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0x98 ..= 0x9F. UTF-8 tail.
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0xA0 ..= 0xA7. UTF-8 tail.
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0xA8 ..= 0xAF. UTF-8 tail.
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0xB0 ..= 0xB7. UTF-8 tail.
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0xB8 ..= 0xBF. UTF-8 tail.
0x81, 0x81, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xC0 ..= 0xC7. Invalid UTF-8; UTF-8-2.
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xC8 ..= 0xCF. UTF-8-2.
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xD0 ..= 0xD7. UTF-8-2.
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // 0xD8 ..= 0xDF. UTF-8-2.
0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, // 0xE0 ..= 0xE7. UTF-8-3.
0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, // 0xE8 ..= 0xEF. UTF-8-3.
0x05, 0x05, 0x05, 0x05, 0x05, 0x81, 0x81, 0x81, // 0xF0 ..= 0xF7. UTF-8-4; Invalid UTF-8.
0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, // 0xF8 ..= 0xFF. Invalid UTF-8.
// 0 1 2 3 4 5 6 7
// 8 9 A B C D E F
]
// lut_classes is:
// - 0x00 (bitmask 0x0001) is CLASS_WHITESPACE.
// - 0x01 (bitmask 0x0002) is CLASS_STRING.
// - 0x02 (bitmask 0x0004) is CLASS_COMMA.
+55 -33
View File
@@ -55,8 +55,9 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
var stack_bit : base.u32[..= 31]
var match : base.u32[..= 2]
var c : base.u8
var class : base.u8[..= 0x0F]
var backslash : base.u8
var char : base.u8
var class : base.u8[..= 0x0F]
var uni4_ok : base.u8
var uni4_string : base.u64
@@ -203,26 +204,28 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
continue.string_loop
}
c = args.src.peek_u8()
char = lut_chars[c]
if c <= 0x22 { // 0x22 is '"'.
if c == 0x22 { // 0x22 is '"'.
if char == 0x00 { // Non-special ASCII.
args.src.skip32_fast!(actual: 1, worst_case: 1)
if string_length >= 0xFFFE {
args.dst.write_fast_token!(
value: 0x20_0220,
length: string_length as base.u64)
value: 0x20_0221,
length: 0xFFFF)
string_length = 0
break.string_loop
} else if c < 0x20 { // 0x20 is ' '.
if string_length > 0 {
args.dst.write_fast_token!(
value: 0x20_0221,
length: string_length as base.u64)
string_length = 0
}
return "#bad input"
continue.string_loop
}
string_length += 1
continue
} else if c == 0x5C { // 0x5C is '\\'.
} else if char == 0x01 { // '"'
args.dst.write_fast_token!(
value: 0x20_0220,
length: string_length as base.u64)
string_length = 0
break.string_loop
} else if char == 0x02 { // '\\'.
if string_length > 0 {
args.dst.write_fast_token!(
value: 0x20_0221,
@@ -236,29 +239,31 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
if args.src.available() < 2 {
if args.src.is_closed() {
return "#bad input"
return "#bad backslash-escape"
}
yield? base."$short read"
string_length = 0
char = 0
continue.string_loop
}
backslash = (args.src.peek_u16le() >> 8) as base.u8
c = lut_backslashes[backslash]
if c > 0 {
c = (args.src.peek_u16le() >> 8) as base.u8
backslash = lut_backslashes[c]
if backslash > 0 {
args.src.skip32_fast!(actual: 2, worst_case: 2)
args.dst.write_fast_token!(
value: 0x40_0000 | ((c & 0x7F) as base.u64),
value: 0x40_0000 | ((backslash & 0x7F) as base.u64),
length: 2)
continue.string_loop
} else if backslash == 0x75 { // 0x75 is 'u'.
} else if c == 0x75 { // 0x75 is 'u'.
// -------- BEGIN backslash-u.
if args.src.available() < 6 {
if args.src.is_closed() {
return "#bad input"
return "#bad backslash-escape"
}
yield? base."$short read"
string_length = 0
char = 0
continue.string_loop
}
@@ -281,7 +286,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
if uni4_ok == 0 {
// It wasn't 4 hexadecimal digits. No-op
// (and fall through to "#bad input").
// (and fall through to "#bad
// backslash-escape").
} else if (uni4_value < 0xD800) or (0xDFFF < uni4_value) {
// Not a Unicode surrogate. We're good.
@@ -293,7 +299,7 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
} else if uni4_value >= 0xDC00 {
// Low surrogate. No-op (and fall through
// to "#bad input").
// to "#bad backslash-escape").
} else {
// High surrogate, which needs to be
@@ -302,11 +308,12 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
// high surrogate. We need another 6.
if args.src.available() < 12 {
if args.src.is_closed() {
return "#bad input"
return "#bad backslash-escape"
}
yield? base."$short read"
string_length = 0
uni4_value = 0
char = 0
continue.string_loop
}
@@ -356,7 +363,8 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
} else {
// Roll back the 4 bytes, then fall
// through to "#bad input".
// through to "#invalid
// backslash-escape".
uni4_rollback = 4
while uni4_rollback > 0 {
uni4_rollback -= 1
@@ -371,18 +379,32 @@ pub func decoder.decode_tokens?(dst: base.token_writer, src: base.io_reader) {
// -------- END backslash-u.
}
return "#bad input"
return "#bad backslash-escape"
} else if char <= 0x10 {
// TODO: reject invalid UTF-8.
args.src.skip32_fast!(actual: 1, worst_case: 1)
if string_length >= 0xFFFE {
args.dst.write_fast_token!(
value: 0x20_0221,
length: 0xFFFF)
string_length = 0
continue.string_loop
}
string_length += 1
continue
}
args.src.skip32_fast!(actual: 1, worst_case: 1)
if string_length >= 0xFFFE {
if string_length > 0 {
args.dst.write_fast_token!(
value: 0x20_0221,
length: 0xFFFF)
length: string_length as base.u64)
string_length = 0
continue.string_loop
}
string_length += 1
if char == 0x80 {
return "#bad C0 control code"
}
return "#bad UTF-8"
}
} endwhile.string_loop
+85 -3
View File
@@ -219,7 +219,7 @@ golden_test json_nobel_prizes_gt = {
// ---------------- JSON Tests
const char* //
test_wuffs_json_decode_tokens() {
test_wuffs_json_decode_interface() {
CHECK_FOCUS(__func__);
wuffs_json__decoder dec;
CHECK_STATUS("initialize",
@@ -375,7 +375,88 @@ test_wuffs_json_decode_unicode4_escapes() {
}
if (total_length != src.meta.ri) {
RETURN_FAIL("%s: total length: have 0x%" PRIu64 ", want 0x%" PRIu64,
RETURN_FAIL("%s: total length: have %" PRIu64 ", want %" PRIu64,
test_cases[tc].str, total_length, src.meta.ri);
}
}
return NULL;
}
const char* //
test_wuffs_json_decode_string() {
CHECK_FOCUS(__func__);
const char* bad_bac = wuffs_json__error__bad_backslash_escape;
const char* bad_ccc = wuffs_json__error__bad_c0_control_code;
const char* bad_utf = wuffs_json__error__bad_utf_8;
struct {
const char* want_status_repr;
const char* str;
} test_cases[] = {
{.want_status_repr = NULL, .str = "\"+++\\\"+\\/+\\\\+++\""},
{.want_status_repr = NULL, .str = "\"+++\\b+\\f+\\n+\\r+\\t+++\""},
{.want_status_repr = NULL, .str = "\"\x20\""}, // U+00000020.
{.want_status_repr = NULL, .str = "\"\xC2\x80\""}, // U+00000080.
{.want_status_repr = NULL, .str = "\"\xDF\xBF\""}, // U+000007FF.
{.want_status_repr = NULL, .str = "\"\xE0\xA0\x80\""}, // U+00000800.
{.want_status_repr = NULL, .str = "\"\xED\xAF\xBF\""}, // U+0000DBFF.
{.want_status_repr = NULL, .str = "\"\xEE\x80\x80\""}, // U+0000E000.
{.want_status_repr = NULL, .str = "\"\xEF\xBF\xBF\""}, // U+0000FFFF.
{.want_status_repr = NULL, .str = "\"\xF0\x90\x80\x80\""}, // U+00010000.
{.want_status_repr = NULL, .str = "\"\xF4\x8F\xBF\xBF\""}, // U+0010FFFF.
{.want_status_repr = NULL, .str = "\"abc\""},
{.want_status_repr = NULL, .str = "\"i\x6Ak\""},
{.want_status_repr = NULL, .str = "\"space+\x20+space\""},
{.want_status_repr = NULL, .str = "\"tab+\\t+tab\""},
{.want_status_repr = NULL, .str = "\"tab+\\u0009+tab\""},
{.want_status_repr = bad_bac, .str = "\"\\uIJKL\""},
{.want_status_repr = bad_bac, .str = "\"space+\\x20+space\""},
{.want_status_repr = bad_ccc, .str = "\"\x1F\""},
{.want_status_repr = bad_ccc, .str = "\"tab+\t+tab\""},
{.want_status_repr = bad_utf, .str = "\"\xF5\""},
{.want_status_repr = bad_utf, .str = "\"\xFF\xFF\xFF\xFF\""},
};
wuffs_json__decoder dec;
int tc;
for (tc = 0; tc < WUFFS_TESTLIB_ARRAY_SIZE(test_cases); tc++) {
CHECK_STATUS("initialize",
wuffs_json__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
wuffs_base__token_buffer tok = ((wuffs_base__token_buffer){
.data = global_have_token_slice,
});
size_t n = strlen(test_cases[tc].str);
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = wuffs_base__make_slice_u8((void*)(test_cases[tc].str), n),
.meta = wuffs_base__make_io_buffer_meta(n, 0, 0, true),
});
wuffs_base__status have_status =
wuffs_json__decoder__decode_tokens(&dec, &tok, &src);
uint64_t total_length = 0;
size_t i;
for (i = tok.meta.ri; i < tok.meta.wi; i++) {
wuffs_base__token* t = &tok.data.ptr[i];
total_length =
wuffs_base__u64__sat_add(total_length, wuffs_base__token__length(t));
}
if (have_status.repr != test_cases[tc].want_status_repr) {
RETURN_FAIL("%s: have \"%s\", want \"%s\"", test_cases[tc].str,
have_status.repr, test_cases[tc].want_status_repr);
}
if (total_length != src.meta.ri) {
RETURN_FAIL("%s: total length: have %" PRIu64 ", want %" PRIu64,
test_cases[tc].str, total_length, src.meta.ri);
}
}
@@ -444,7 +525,8 @@ proc tests[] = {
test_strconv_parse_number_i64, //
test_strconv_parse_number_u64, //
test_wuffs_json_decode_tokens, //
test_wuffs_json_decode_interface, //
test_wuffs_json_decode_string, //
test_wuffs_json_decode_unicode4_escapes, //
#ifdef WUFFS_MIMIC