diff --git a/internal/cgen/base/token-public.h b/internal/cgen/base/token-public.h index 1343ab97..9b863248 100644 --- a/internal/cgen/base/token-public.h +++ b/internal/cgen/base/token-public.h @@ -87,6 +87,8 @@ wuffs_base__make_token(uint64_t repr) { // -------- +#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF + #define WUFFS_BASE__TOKEN__VALUE__MASK 0x3FFFFFFFFFFF #define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0x3FFFFF #define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF diff --git a/internal/cgen/data.go b/internal/cgen/data.go index 65769c73..5271f03a 100644 --- a/internal/cgen/data.go +++ b/internal/cgen/data.go @@ -312,7 +312,7 @@ const baseTokenPublicH = "" + "tail).\n //\n // The value_major is a 21-bit [Base38](doc/note/base38-and-fourcc.md) value.\n // If all of its bits are zero (special cased for Wuffs' built-in \"base\"\n // package) then the value_minor is further sub-divided:\n // - Bits 41 .. 39 ( 3 bits) is the VBC (value_base_category).\n // - Bits 38 .. 18 (21 bits) is the VBD (value_base_detail).\n //\n // The high 46 bits (bits 63 .. 18) only have VBC and VBD semantics when the\n // high 22 bits (the value_major) are all zero. An equivalent test is that\n // the high 25 bits (the notional VBC) has a numerical value less than 8.\n //\n // At 21 bits, the VBD can hold every valid Unicode code point.\n //\n // If value_major is non-zero then value_minor has whatever arbitrary meaning\n // the tokenizer's package assigns to it.\n //\n // Multiple consecutive tokens can form a larger conceptual unit. For\n // example, an \"abc\\tz\" string is a single higher level concept but at the\n // lower level, it could consist of multiple tokens: the quotes '\"', the\n " + "// ASCII texts \"abc\" and \"z\" and the backslash-escaped tab '\\t'. The LP and\n // LN (link_prev and link_next) bits denote tokens that are part of a\n // multi-token chain:\n // - LP means that this token is not the first (there is a previous token).\n // - LN means that this token is not the last (there is a next token).\n //\n // In particular, a stand-alone token will have both link bits set to zero.\n uint64_t repr;\n\n#ifdef __cplusplus\n inline uint64_t value() const;\n inline uint64_t value_major() const;\n inline uint64_t value_minor() const;\n inline uint64_t value_base_category() const;\n inline uint64_t value_base_detail() const;\n inline bool link_prev() const;\n inline bool link_next() const;\n inline uint64_t length() const;\n#endif // __cplusplus\n\n} wuffs_base__token;\n\nstatic inline wuffs_base__token //\nwuffs_base__make_token(uint64_t repr) {\n wuffs_base__token ret;\n ret.repr = repr;\n return ret;\n}\n\n " + "" + - "// --------\n\n#define WUFFS_BASE__TOKEN__VALUE__MASK 0x3FFFFFFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0x3FFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x1FFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF\n#define WUFFS_BASE__TOKEN__LINK__MASK 0x3\n#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 39\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 18\n#define WUFFS_BASE__TOKEN__LINK__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n#define WUFFS_BASE__TOKEN__LINK_PREV 0x20000\n#define WUFFS_BASE__TOKEN__LINK_NEXT 0x10000\n\n " + + "// --------\n\n#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__MASK 0x3FFFFFFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0x3FFFFF\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__MASK 0x1FFFFFF\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__MASK 0x1FFFFF\n#define WUFFS_BASE__TOKEN__LINK__MASK 0x3\n#define WUFFS_BASE__TOKEN__LENGTH__MASK 0xFFFF\n\n#define WUFFS_BASE__TOKEN__VALUE__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_MAJOR__SHIFT 42\n#define WUFFS_BASE__TOKEN__VALUE_MINOR__SHIFT 18\n#define WUFFS_BASE__TOKEN__VALUE_BASE_CATEGORY__SHIFT 39\n#define WUFFS_BASE__TOKEN__VALUE_BASE_DETAIL__SHIFT 18\n#define WUFFS_BASE__TOKEN__LINK__SHIFT 16\n#define WUFFS_BASE__TOKEN__LENGTH__SHIFT 0\n\n#define WUFFS_BASE__TOKEN__LINK_PREV 0x20000\n#define WUFFS_BASE__TOKEN__LINK_NEXT 0x10000\n\n " + "" + "// --------\n\n#define WUFFS_BASE__TOKEN__VBC__FILLER 0\n#define WUFFS_BASE__TOKEN__VBC__STRUCTURE 1\n#define WUFFS_BASE__TOKEN__VBC__STRING 2\n#define WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT 3\n#define WUFFS_BASE__TOKEN__VBC__LITERAL 4\n#define WUFFS_BASE__TOKEN__VBC__NUMBER 5\n\n " + "" + diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 5c03b203..2c66a3b9 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -1803,6 +1803,8 @@ wuffs_base__make_token(uint64_t repr) { // -------- +#define WUFFS_BASE__TOKEN__LENGTH__MAX_INCL 0xFFFF + #define WUFFS_BASE__TOKEN__VALUE__MASK 0x3FFFFFFFFFFF #define WUFFS_BASE__TOKEN__VALUE_MAJOR__MASK 0x3FFFFF #define WUFFS_BASE__TOKEN__VALUE_MINOR__MASK 0xFFFFFF diff --git a/test/c/std/json.c b/test/c/std/json.c index 8253ec29..15795297 100644 --- a/test/c/std/json.c +++ b/test/c/std/json.c @@ -394,6 +394,157 @@ wuffs_json_decode(wuffs_base__token_buffer* tok, } } +// test_wuffs_json_decode_prior_valid_utf_8 tests that when encountering +// invalid or incomplete UTF-8, or a backslash-escape, any prior valid UTF-8 is +// still output. The decoder batches output so that, ignoring the quotation +// marks, "abc\xCE\x94efg" can be a single 8-length token instead of multiple +// (e.g. 3+2+3) tokens. On the other hand, while "abc\xFF" ends with one byte +// of invalid UTF-8, the 3 good bytes before that should still be output. +const char* // +test_wuffs_json_decode_prior_valid_utf_8() { + CHECK_FOCUS(__func__); + + // The test cases contain combinations of valid, partial and invalid UTF-8: + // - "\xCE\x94" is U+00000394 GREEK CAPITAL LETTER DELTA. + // - "\xE2\x98\x83" is U+00002603 SNOWMAN. + // - "\xF0\x9F\x92\xA9" is U+0001F4A9 PILE OF POO. + // + // The code below can also add trailing 's' bytes, which change e.g. the + // partial multi-byte UTF-8 "\xE2" to be the invalid UTF-8 "\xE2s". + const char* test_cases[] = { + "", + "\\t", + "\\u", + "\\u1234", + "\x1F", // Valid UTF-8 but invalid in a JSON string. + "\x20", + "\xCE", + "\xCE\x94", + "\xE2", + "\xE2\x98", + "\xE2\x98\x83", + "\xE2\x98\x83\xCE", + "\xE2\x98\x83\xCE\x94", + "\xF0", + "\xF0\x9F", + "\xF0\x9F\x92", + "\xF0\x9F\x92\xA9", + "\xF0\x9F\x92\xA9\xCE", + "\xF0\x9F\x92\xA9\xCE\x94", + }; + + size_t prefixes[] = { + 0, + 1, + 15, + WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 9, + WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 8, + WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 7, + WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 6, + WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 5, + WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 4, + WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 3, + WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 2, + WUFFS_BASE__TOKEN__LENGTH__MAX_INCL - 1, + WUFFS_BASE__TOKEN__LENGTH__MAX_INCL + 0, + }; + + size_t suffixes[] = {0, 1, 17}; + + wuffs_json__decoder dec; + + int tc; + for (tc = 0; tc < WUFFS_TESTLIB_ARRAY_SIZE(test_cases); tc++) { + size_t n = strlen(test_cases[tc]); + size_t num_preceding = 0; + while (num_preceding < n) { + wuffs_base__utf_8__next__output x = + wuffs_base__utf_8__next(wuffs_base__make_slice_u8( + (void*)(test_cases[tc]) + num_preceding, n - num_preceding)); + if (!wuffs_base__utf_8__next__output__is_valid(&x) || + (x.code_point < 0x20) || (x.code_point == '\\')) { + break; + } + num_preceding += x.byte_length; + if (num_preceding > n) { + RETURN_FAIL("tc=%d: utf_8__next overflow", tc); + } + } + + int pre; + for (pre = 0; pre < WUFFS_TESTLIB_ARRAY_SIZE(prefixes); pre++) { + size_t prefix = prefixes[pre]; + + int suf; + for (suf = 0; suf < WUFFS_TESTLIB_ARRAY_SIZE(suffixes); suf++) { + size_t suffix = suffixes[suf]; + + // Set src to "\"ppp...pppMIDDLEsss...sss", with a leading quotation + // mark, where prefix and suffix are the number of 'p's and 's's and + // test_cases[tc] is the "MIDDLE". + wuffs_base__slice_u8 src_data = ((wuffs_base__slice_u8){ + .ptr = global_src_array, + .len = 1 + prefix + n + suffix, + }); + if (src_data.len > IO_BUFFER_SIZE) { + RETURN_FAIL("total src length is too long"); + } + src_data.ptr[0] = '\"'; + memset(&src_data.ptr[1], 'p', prefix); + memcpy(&src_data.ptr[1 + prefix], test_cases[tc], n); + memset(&src_data.ptr[1 + prefix + n], 's', suffix); + + int closed; + for (closed = 0; closed < 2; closed++) { + CHECK_STATUS( + "initialize", + wuffs_json__decoder__initialize( + &dec, sizeof dec, WUFFS_VERSION, + WUFFS_INITIALIZE__LEAVE_INTERNAL_BUFFERS_UNINITIALIZED)); + + wuffs_base__token_buffer tok = ((wuffs_base__token_buffer){ + .data = global_have_token_slice, + }); + + wuffs_base__io_buffer src = ((wuffs_base__io_buffer){ + .data = src_data, + .meta = wuffs_base__make_io_buffer_meta(src_data.len, 0, 0, + closed != 0), + }); + + wuffs_json__decoder__decode_tokens(&dec, &tok, &src); + + size_t have = 0; + while (tok.meta.ri < tok.meta.wi) { + wuffs_base__token* t = &tok.data.ptr[tok.meta.ri++]; + uint64_t vbc = wuffs_base__token__value_base_category(t); + if (vbc == WUFFS_BASE__TOKEN__VBC__UNICODE_CODE_POINT) { + break; + } else if (vbc == WUFFS_BASE__TOKEN__VBC__STRING) { + have += wuffs_base__token__length(t); + } else { + RETURN_FAIL( + "tc=%d, prefix=%zu, suffix=%zu, closed=%d: unexpected token", + tc, prefix, suffix, closed); + } + } + size_t want = 1 + prefix + num_preceding; // 1 for the leading '\"'. + if (num_preceding == n) { + want += suffix; + } + if (have != want) { + RETURN_FAIL( + "tc=%d, prefix=%zu, suffix=%zu, closed=%d: have %zu, want %zu", + tc, prefix, suffix, closed, have, want); + } + } + } + } + } + + return NULL; +} + const char* // test_wuffs_json_decode_unicode4_escapes() { CHECK_FOCUS(__func__); @@ -673,9 +824,10 @@ proc tests[] = { test_strconv_parse_number_u64, // test_strconv_utf_8_next, // - test_wuffs_json_decode_interface, // - test_wuffs_json_decode_string, // - test_wuffs_json_decode_unicode4_escapes, // + test_wuffs_json_decode_interface, // + test_wuffs_json_decode_prior_valid_utf_8, // + test_wuffs_json_decode_string, // + test_wuffs_json_decode_unicode4_escapes, // #ifdef WUFFS_MIMIC