Test std/json emit pending strings on bad UTF-8

This commit is contained in:
Nigel Tao committed 2020-03-02 13:59:46 +11:00
1 parent 19967648ef
commit 43f0a87a6d
4 files changed
+160 -4

No files matched your search

+2
View File
@@ -87,6 +87,8 @@ wuffs_base__make_token(uint64_t repr) {
// --------
#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF
#define WUFFS_BASE__TOKEN__VALUE__MASK 0x3FFFFFFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0x3FFFFF
#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF
+1 -1
View File
@@ -312,7 +312,7 @@ const baseTokenPublicH = "" +
"tail).\n //\n // The value_major is a 21-bit [Base38](doc/note/base38-and-fourcc.md) value.\n // If all of its bits are zero (special cased for Wuffs' built-in \"base\"\n // package) then the value_minor is further sub-divided:\n // - Bits 41 .. 39 ( 3 bits) is the VBC (value_base_category).\n // - Bits 38 .. 18 (21 bits) is the VBD (value_base_detail).\n //\n // The high 46 bits (bits 63 .. 18) only have VBC and VBD semantics when the\n // high 22 bits (the value_major) are all zero. An equivalent test is that\n // the high 25 bits (the notional VBC) has a numerical value less than 8.\n //\n // At 21 bits, the VBD can hold every valid Unicode code point.\n //\n // If value_major is non-zero then value_minor has whatever arbitrary meaning\n // the tokenizer's package assigns to it.\n //\n // Multiple consecutive tokens can form a larger conceptual unit. For\n // example, an \"abc\\tz\" string is a single higher level concept but at the\n // lower level, it could consist of multiple tokens: the quotes '\"', the\n " +
"// ASCII texts \"abc\" and \"z\" and the backslash-escaped tab '\\t'. The LP and\n // LN (link_prev and link_next) bits denote tokens that are part of a\n // multi-token chain:\n // - LP means that this token is not the first (there is a previous token).\n // - LN means that this token is not the last (there is a next token).\n //\n // In particular, a stand-alone token will have both link bits set to zero.\n uint64_t repr;\n\n#ifdef __cplusplus\n inline uint64_t value() const;\n inline uint64_t value_major() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_category() const;\n inline uint64_t value_base_detail() const;\n inline bool link_prev() const;\n inline bool link_next() const;\n inline uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\n " +
"" +
"// --------\n\n#define WUFFS_BASE__TOKEN__VALUE__MASK 0x3FFFFFFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0x3FFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x1FFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF\n#define WUFFS_BASE__TOKEN__LINK__MASK 0x3\n#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 39\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 18\n#define WUFFS_BASE__TOKEN__LINK__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n#define WUFFS_BASE__TOKEN__LINK_PREV 0x20000\n#define WUFFS_BASE__TOKEN__LINK_NEXT 0x10000\n\n " +
"// --------\n\n#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__MASK 0x3FFFFFFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0x3FFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x1FFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF\n#define WUFFS_BASE__TOKEN__LINK__MASK 0x3\n#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 39\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 18\n#define WUFFS_BASE__TOKEN__LINK__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n#define WUFFS_BASE__TOKEN__LINK_PREV 0x20000\n#define WUFFS_BASE__TOKEN__LINK_NEXT 0x10000\n\n " +
"" +
"// --------\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1\n#define WUFFS_BASE__TOKEN__VBC__STRING 2\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 3\n#define WUFFS_BASE__TOKEN__VBC__LITERAL 4\n#define WUFFS_BASE__TOKEN__VBC__NUMBER 5\n\n " +
"" +
+2
View File
@@ -1803,6 +1803,8 @@ wuffs_base__make_token(uint64_t repr) {
// --------
#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF
#define WUFFS_BASE__TOKEN__VALUE__MASK 0x3FFFFFFFFFFF
#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0x3FFFFF
#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF
+155 -3
View File
@@ -394,6 +394,157 @@ wuffs_json_decode(wuffs_base__token_buffer* tok,
}
}
// test_wuffs_json_decode_prior_valid_utf_8 tests that when encountering
// invalid or incomplete UTF-8, or a backslash-escape, any prior valid UTF-8 is
// still output. The decoder batches output so that, ignoring the quotation
// marks, "abc\xCE\x94efg" can be a single 8-length token instead of multiple
// (e.g. 3+2+3) tokens. On the other hand, while "abc\xFF" ends with one byte
// of invalid UTF-8, the 3 good bytes before that should still be output.
const char* //
test_wuffs_json_decode_prior_valid_utf_8() {
CHECK_FOCUS(__func__);
// The test cases contain combinations of valid, partial and invalid UTF-8:
// - "\xCE\x94" is U+00000394 GREEK CAPITAL LETTER DELTA.
// - "\xE2\x98\x83" is U+00002603 SNOWMAN.
// - "\xF0\x9F\x92\xA9" is U+0001F4A9 PILE OF POO.
//
// The code below can also add trailing 's' bytes, which change e.g. the
// partial multi-byte UTF-8 "\xE2" to be the invalid UTF-8 "\xE2s".
const char* test_cases[] = {
"",
"\\t",
"\\u",
"\\u1234",
"\x1F", // Valid UTF-8 but invalid in a JSON string.
"\x20",
"\xCE",
"\xCE\x94",
"\xE2",
"\xE2\x98",
"\xE2\x98\x83",
"\xE2\x98\x83\xCE",
"\xE2\x98\x83\xCE\x94",
"\xF0",
"\xF0\x9F",
"\xF0\x9F\x92",
"\xF0\x9F\x92\xA9",
"\xF0\x9F\x92\xA9\xCE",
"\xF0\x9F\x92\xA9\xCE\x94",
};
size_t prefixes[] = {
0,
1,
15,
WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 9,
WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 8,
WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 7,
WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 6,
WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 5,
WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 4,
WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 3,
WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 2,
WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 1,
WUFFS_BASE__TOKEN__LENGTH__MAX_INCL + 0,
};
size_t suffixes[] = {0, 1, 17};
wuffs_json__decoder dec;
int tc;
for (tc = 0; tc < WUFFS_TESTLIB_ARRAY_SIZE(test_cases); tc++) {
size_t n = strlen(test_cases[tc]);
size_t num_preceding = 0;
while (num_preceding < n) {
wuffs_base__utf_8__next__output x =
wuffs_base__utf_8__next(wuffs_base__make_slice_u8(
(void*)(test_cases[tc]) + num_preceding, n - num_preceding));
if (!wuffs_base__utf_8__next__output__is_valid(&x) ||
(x.code_point < 0x20) || (x.code_point == '\\')) {
break;
}
num_preceding += x.byte_length;
if (num_preceding > n) {
RETURN_FAIL("tc=%d: utf_8__next overflow", tc);
}
}
int pre;
for (pre = 0; pre < WUFFS_TESTLIB_ARRAY_SIZE(prefixes); pre++) {
size_t prefix = prefixes[pre];
int suf;
for (suf = 0; suf < WUFFS_TESTLIB_ARRAY_SIZE(suffixes); suf++) {
size_t suffix = suffixes[suf];
// Set src to "\"ppp...pppMIDDLEsss...sss", with a leading quotation
// mark, where prefix and suffix are the number of 'p's and 's's and
// test_cases[tc] is the "MIDDLE".
wuffs_base__slice_u8 src_data = ((wuffs_base__slice_u8){
.ptr = global_src_array,
.len = 1 + prefix + n + suffix,
});
if (src_data.len > IO_BUFFER_SIZE) {
RETURN_FAIL("total src length is too long");
}
src_data.ptr[0] = '\"';
memset(&src_data.ptr[1], 'p', prefix);
memcpy(&src_data.ptr[1 + prefix], test_cases[tc], n);
memset(&src_data.ptr[1 + prefix + n], 's', suffix);
int closed;
for (closed = 0; closed < 2; closed++) {
CHECK_STATUS(
"initialize",
wuffs_json__decoder__initialize(
&dec, sizeof dec, WUFFS_VERSION,
WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED));
wuffs_base__token_buffer tok = ((wuffs_base__token_buffer){
.data = global_have_token_slice,
});
wuffs_base__io_buffer src = ((wuffs_base__io_buffer){
.data = src_data,
.meta = wuffs_base__make_io_buffer_meta(src_data.len, 0, 0,
closed != 0),
});
wuffs_json__decoder__decode_tokens(&dec, &tok, &src);
size_t have = 0;
while (tok.meta.ri < tok.meta.wi) {
wuffs_base__token* t = &tok.data.ptr[tok.meta.ri++];
uint64_t vbc = wuffs_base__token__value_base_category(t);
if (vbc == WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT) {
break;
} else if (vbc == WUFFS_BASE__TOKEN__VBC__STRING) {
have += wuffs_base__token__length(t);
} else {
RETURN_FAIL(
"tc=%d, prefix=%zu, suffix=%zu, closed=%d: unexpected token",
tc, prefix, suffix, closed);
}
}
size_t want = 1 + prefix + num_preceding; // 1 for the leading '\"'.
if (num_preceding == n) {
want += suffix;
}
if (have != want) {
RETURN_FAIL(
"tc=%d, prefix=%zu, suffix=%zu, closed=%d: have %zu, want %zu",
tc, prefix, suffix, closed, have, want);
}
}
}
}
}
return NULL;
}
const char* //
test_wuffs_json_decode_unicode4_escapes() {
CHECK_FOCUS(__func__);
@@ -673,9 +824,10 @@ proc tests[] = {
test_strconv_parse_number_u64, //
test_strconv_utf_8_next, //
test_wuffs_json_decode_interface, //
test_wuffs_json_decode_string, //
test_wuffs_json_decode_unicode4_escapes, //
test_wuffs_json_decode_interface, //
test_wuffs_json_decode_prior_valid_utf_8, //
test_wuffs_json_decode_string, //
test_wuffs_json_decode_unicode4_escapes, //
#ifdef WUFFS_MIMIC