mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-18 00:32:11 +01:00
Test std/json emit pending strings on bad UTF-8
This commit is contained in:
1 parent
19967648ef
commit
43f0a87a6d
4 files changed
+160
-4
No files matched your search
@@ -87,6 +87,8 @@ wuffs_base__make_token(uint64_t repr) {
|
||||
|
||||
// --------
|
||||
|
||||
#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF
|
||||
|
||||
#define WUFFS_BASE__TOKEN__VALUE__MASK 0x3FFFFFFFFFFF
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0x3FFFFF
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF
|
||||
|
||||
@@ -312,7 +312,7 @@ const baseTokenPublicH = "" +
|
||||
"tail).\n //\n // The value_major is a 21-bit [Base38](doc/note/base38-and-fourcc.md) value.\n // If all of its bits are zero (special cased for Wuffs' built-in \"base\"\n // package) then the value_minor is further sub-divided:\n // - Bits 41 .. 39 ( 3 bits) is the VBC (value_base_category).\n // - Bits 38 .. 18 (21 bits) is the VBD (value_base_detail).\n //\n // The high 46 bits (bits 63 .. 18) only have VBC and VBD semantics when the\n // high 22 bits (the value_major) are all zero. An equivalent test is that\n // the high 25 bits (the notional VBC) has a numerical value less than 8.\n //\n // At 21 bits, the VBD can hold every valid Unicode code point.\n //\n // If value_major is non-zero then value_minor has whatever arbitrary meaning\n // the tokenizer's package assigns to it.\n //\n // Multiple consecutive tokens can form a larger conceptual unit. For\n // example, an \"abc\\tz\" string is a single higher level concept but at the\n // lower level, it could consist of multiple tokens: the quotes '\"', the\n " +
|
||||
"// ASCII texts \"abc\" and \"z\" and the backslash-escaped tab '\\t'. The LP and\n // LN (link_prev and link_next) bits denote tokens that are part of a\n // multi-token chain:\n // - LP means that this token is not the first (there is a previous token).\n // - LN means that this token is not the last (there is a next token).\n //\n // In particular, a stand-alone token will have both link bits set to zero.\n uint64_t repr;\n\n#ifdef __cplusplus\n inline uint64_t value() const;\n inline uint64_t value_major() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_category() const;\n inline uint64_t value_base_detail() const;\n inline bool link_prev() const;\n inline bool link_next() const;\n inline uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\n " +
|
||||
"" +
|
||||
"// --------\n\n#define WUFFS_BASE__TOKEN__VALUE__MASK 0x3FFFFFFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0x3FFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x1FFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF\n#define WUFFS_BASE__TOKEN__LINK__MASK 0x3\n#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 39\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 18\n#define WUFFS_BASE__TOKEN__LINK__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n#define WUFFS_BASE__TOKEN__LINK_PREV 0x20000\n#define WUFFS_BASE__TOKEN__LINK_NEXT 0x10000\n\n " +
|
||||
"// --------\n\n#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__MASK 0x3FFFFFFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0x3FFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x1FFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF\n#define WUFFS_BASE__TOKEN__LINK__MASK 0x3\n#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 39\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 18\n#define WUFFS_BASE__TOKEN__LINK__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n#define WUFFS_BASE__TOKEN__LINK_PREV 0x20000\n#define WUFFS_BASE__TOKEN__LINK_NEXT 0x10000\n\n " +
|
||||
"" +
|
||||
"// --------\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1\n#define WUFFS_BASE__TOKEN__VBC__STRING 2\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 3\n#define WUFFS_BASE__TOKEN__VBC__LITERAL 4\n#define WUFFS_BASE__TOKEN__VBC__NUMBER 5\n\n " +
|
||||
"" +
|
||||
|
||||
@@ -1803,6 +1803,8 @@ wuffs_base__make_token(uint64_t repr) {
|
||||
|
||||
// --------
|
||||
|
||||
#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF
|
||||
|
||||
#define WUFFS_BASE__TOKEN__VALUE__MASK 0x3FFFFFFFFFFF
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0x3FFFFF
|
||||
#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF
|
||||
|
||||
+155
-3
@@ -394,6 +394,157 @@ wuffs_json_decode(wuffs_base__token_buffer* tok,
|
||||
}
|
||||
}
|
||||
|
||||
// test_wuffs_json_decode_prior_valid_utf_8 tests that when encountering
|
||||
// invalid or incomplete UTF-8, or a backslash-escape, any prior valid UTF-8 is
|
||||
// still output. The decoder batches output so that, ignoring the quotation
|
||||
// marks, "abc\xCE\x94efg" can be a single 8-length token instead of multiple
|
||||
// (e.g. 3+2+3) tokens. On the other hand, while "abc\xFF" ends with one byte
|
||||
// of invalid UTF-8, the 3 good bytes before that should still be output.
|
||||
const char* //
|
||||
test_wuffs_json_decode_prior_valid_utf_8() {
|
||||
CHECK_FOCUS(__func__);
|
||||
|
||||
// The test cases contain combinations of valid, partial and invalid UTF-8:
|
||||
// - "\xCE\x94" is U+00000394 GREEK CAPITAL LETTER DELTA.
|
||||
// - "\xE2\x98\x83" is U+00002603 SNOWMAN.
|
||||
// - "\xF0\x9F\x92\xA9" is U+0001F4A9 PILE OF POO.
|
||||
//
|
||||
// The code below can also add trailing 's' bytes, which change e.g. the
|
||||
// partial multi-byte UTF-8 "\xE2" to be the invalid UTF-8 "\xE2s".
|
||||
const char* test_cases[] = {
|
||||
"",
|
||||
"\\t",
|
||||
"\\u",
|
||||
"\\u1234",
|
||||
"\x1F", // Valid UTF-8 but invalid in a JSON string.
|
||||
"\x20",
|
||||
"\xCE",
|
||||
"\xCE\x94",
|
||||
"\xE2",
|
||||
"\xE2\x98",
|
||||
"\xE2\x98\x83",
|
||||
"\xE2\x98\x83\xCE",
|
||||
"\xE2\x98\x83\xCE\x94",
|
||||
"\xF0",
|
||||
"\xF0\x9F",
|
||||
"\xF0\x9F\x92",
|
||||
"\xF0\x9F\x92\xA9",
|
||||
"\xF0\x9F\x92\xA9\xCE",
|
||||
"\xF0\x9F\x92\xA9\xCE\x94",
|
||||
};
|
||||
|
||||
size_t prefixes[] = {
|
||||
0,
|
||||
1,
|
||||
15,
|
||||
WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 9,
|
||||
WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 8,
|
||||
WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 7,
|
||||
WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 6,
|
||||
WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 5,
|
||||
WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 4,
|
||||
WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 3,
|
||||
WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 2,
|
||||
WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 1,
|
||||
WUFFS_BASE__TOKEN__LENGTH__MAX_INCL + 0,
|
||||
};
|
||||
|
||||
size_t suffixes[] = {0, 1, 17};
|
||||
|
||||
wuffs_json__decoder dec;
|
||||
|
||||
int tc;
|
||||
for (tc = 0; tc < WUFFS_TESTLIB_ARRAY_SIZE(test_cases); tc++) {
|
||||
size_t n = strlen(test_cases[tc]);
|
||||
size_t num_preceding = 0;
|
||||
while (num_preceding < n) {
|
||||
wuffs_base__utf_8__next__output x =
|
||||
wuffs_base__utf_8__next(wuffs_base__make_slice_u8(
|
||||
(void*)(test_cases[tc]) + num_preceding, n - num_preceding));
|
||||
if (!wuffs_base__utf_8__next__output__is_valid(&x) ||
|
||||
(x.code_point < 0x20) || (x.code_point == '\\')) {
|
||||
break;
|
||||
}
|
||||
num_preceding += x.byte_length;
|
||||
if (num_preceding > n) {
|
||||
RETURN_FAIL("tc=%d: utf_8__next overflow", tc);
|
||||
}
|
||||
}
|
||||
|
||||
int pre;
|
||||
for (pre = 0; pre < WUFFS_TESTLIB_ARRAY_SIZE(prefixes); pre++) {
|
||||
size_t prefix = prefixes[pre];
|
||||
|
||||
int suf;
|
||||
for (suf = 0; suf < WUFFS_TESTLIB_ARRAY_SIZE(suffixes); suf++) {
|
||||
size_t suffix = suffixes[suf];
|
||||
|
||||
// Set src to "\"ppp...pppMIDDLEsss...sss", with a leading quotation
|
||||
// mark, where prefix and suffix are the number of 'p's and 's's and
|
||||
// test_cases[tc] is the "MIDDLE".
|
||||
wuffs_base__slice_u8 src_data = ((wuffs_base__slice_u8){
|
||||
.ptr = global_src_array,
|
||||
.len = 1 + prefix + n + suffix,
|
||||
});
|
||||
if (src_data.len > IO_BUFFER_SIZE) {
|
||||
RETURN_FAIL("total src length is too long");
|
||||
}
|
||||
src_data.ptr[0] = '\"';
|
||||
memset(&src_data.ptr[1], 'p', prefix);
|
||||
memcpy(&src_data.ptr[1 + prefix], test_cases[tc], n);
|
||||
memset(&src_data.ptr[1 + prefix + n], 's', suffix);
|
||||
|
||||
int closed;
|
||||
for (closed = 0; closed < 2; closed++) {
|
||||
CHECK_STATUS(
|
||||
"initialize",
|
||||
wuffs_json__decoder__initialize(
|
||||
&dec, sizeof dec, WUFFS_VERSION,
|
||||
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
|
||||
|
||||
wuffs_base__token_buffer tok = ((wuffs_base__token_buffer){
|
||||
.data = global_have_token_slice,
|
||||
});
|
||||
|
||||
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
|
||||
.data = src_data,
|
||||
.meta = wuffs_base__make_io_buffer_meta(src_data.len, 0, 0,
|
||||
closed != 0),
|
||||
});
|
||||
|
||||
wuffs_json__decoder__decode_tokens(&dec, &tok, &src);
|
||||
|
||||
size_t have = 0;
|
||||
while (tok.meta.ri < tok.meta.wi) {
|
||||
wuffs_base__token* t = &tok.data.ptr[tok.meta.ri++];
|
||||
uint64_t vbc = wuffs_base__token__value_base_category(t);
|
||||
if (vbc == WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT) {
|
||||
break;
|
||||
} else if (vbc == WUFFS_BASE__TOKEN__VBC__STRING) {
|
||||
have += wuffs_base__token__length(t);
|
||||
} else {
|
||||
RETURN_FAIL(
|
||||
"tc=%d, prefix=%zu, suffix=%zu, closed=%d: unexpected token",
|
||||
tc, prefix, suffix, closed);
|
||||
}
|
||||
}
|
||||
size_t want = 1 + prefix + num_preceding; // 1 for the leading '\"'.
|
||||
if (num_preceding == n) {
|
||||
want += suffix;
|
||||
}
|
||||
if (have != want) {
|
||||
RETURN_FAIL(
|
||||
"tc=%d, prefix=%zu, suffix=%zu, closed=%d: have %zu, want %zu",
|
||||
tc, prefix, suffix, closed, have, want);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const char* //
|
||||
test_wuffs_json_decode_unicode4_escapes() {
|
||||
CHECK_FOCUS(__func__);
|
||||
@@ -673,9 +824,10 @@ proc tests[] = {
|
||||
test_strconv_parse_number_u64, //
|
||||
test_strconv_utf_8_next, //
|
||||
|
||||
test_wuffs_json_decode_interface, //
|
||||
test_wuffs_json_decode_string, //
|
||||
test_wuffs_json_decode_unicode4_escapes, //
|
||||
test_wuffs_json_decode_interface, //
|
||||
test_wuffs_json_decode_prior_valid_utf_8, //
|
||||
test_wuffs_json_decode_string, //
|
||||
test_wuffs_json_decode_unicode4_escapes, //
|
||||
|
||||
#ifdef WUFFS_MIMIC
|
||||
|
||||
|
||||
Reference in new issue
Block a user