From 809220e6ccd20cc11e9d2ba35ad310c95eda27bc Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Thu, 27 Feb 2020 14:23:10 +1100 Subject: [PATCH] Add wuffs_base__utf_8__next --- doc/changelog.md | 2 + internal/cgen/base/strconv-impl.c | 146 +++++++++++++ internal/cgen/base/strconv-private.h | 2 + internal/cgen/base/strconv-public.h | 123 +++++++++++ internal/cgen/data.go | 25 ++- release/c/wuffs-unsupported-snapshot.c | 271 +++++++++++++++++++++++++ test/c/std/json.c | 110 ++++++++++ 7 files changed, 676 insertions(+), 3 deletions(-) diff --git a/doc/changelog.md b/doc/changelog.md index d86fa041..c6190d67 100644 --- a/doc/changelog.md +++ b/doc/changelog.md @@ -3,6 +3,8 @@ ## Work In Progress +- Added `base` library support for `atoi`-like string conversion. +- Added `base` library support for UTF-8. - Added `endwhile` syntax. - Added `example/imageviewer`. - Added `example/jsonptr`. diff --git a/internal/cgen/base/strconv-impl.c b/internal/cgen/base/strconv-impl.c index bc39b515..be5aa9af 100644 --- a/internal/cgen/base/strconv-impl.c +++ b/internal/cgen/base/strconv-impl.c @@ -298,3 +298,149 @@ fail_out_of_bounds: return ret; } while (0); } + +// ---------------- Unicode and UTF-8 + +// wuffs_base__utf_8__byte_length_minus_1 is the byte length (minus 1) of a +// UTF-8 encoded code point, based on the encoding's initial byte. +// - 0x00 is 1-byte UTF-8 (ASCII). +// - 0x01 is the start of 2-byte UTF-8. +// - 0x02 is the start of 3-byte UTF-8. +// - 0x03 is the start of 4-byte UTF-8. +// - 0x40 is a UTF-8 tail byte. +// - 0x80 is invalid UTF-8. +// +// RFC 3629 (UTF-8) gives this grammar for valid UTF-8: +// UTF8-1 = %x00-7F +// UTF8-2 = %xC2-DF UTF8-tail +// UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / +// %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) +// UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / +// %xF4 %x80-8F 2( UTF8-tail ) +// UTF8-tail = %x80-BF +static const uint8_t wuffs_base__utf_8__byte_length_minus_1[256] = { + // 0 1 2 3 4 5 6 7 + // 8 9 A B C D E F + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x00 ..= 0x07. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x08 ..= 0x0F. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x10 ..= 0x17. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x18 ..= 0x1F. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20 ..= 0x27. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x28 ..= 0x2F. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x30 ..= 0x37. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x38 ..= 0x3F. + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x40 ..= 0x47. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x48 ..= 0x4F. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x50 ..= 0x57. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x58 ..= 0x5F. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x60 ..= 0x67. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x68 ..= 0x6F. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x70 ..= 0x77. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x78 ..= 0x7F. + + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x80 ..= 0x87. + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x88 ..= 0x8F. + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x90 ..= 0x97. + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x98 ..= 0x9F. + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xA0 ..= 0xA7. + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xA8 ..= 0xAF. + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xB0 ..= 0xB7. + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xB8 ..= 0xBF. + + 0x80, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xC0 ..= 0xC7. + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xC8 ..= 0xCF. + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xD0 ..= 0xD7. + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xD8 ..= 0xDF. + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, // 0xE0 ..= 0xE7. + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, // 0xE8 ..= 0xEF. + 0x03, 0x03, 0x03, 0x03, 0x03, 0x80, 0x80, 0x80, // 0xF0 ..= 0xF7. + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xF8 ..= 0xFF. + // 0 1 2 3 4 5 6 7 + // 8 9 A B C D E F +}; + +wuffs_base__utf_8__next__output // +wuffs_base__utf_8__next(wuffs_base__slice_u8 s) { + if (s.len == 0) { + return wuffs_base__make_utf_8__next__output(0, 0); + } + uint32_t c = s.ptr[0]; + switch (wuffs_base__utf_8__byte_length_minus_1[c & 0xFF]) { + case 0: + return wuffs_base__make_utf_8__next__output(c, 1); + + case 1: + if (s.len < 2) { + break; + } + c = wuffs_base__load_u16le__no_bounds_check(s.ptr); + if ((c & 0xC000) != 0x8000) { + break; + } + c = (0x0007C0 & (c << 6)) | (0x00003F & (c >> 8)); + return wuffs_base__make_utf_8__next__output(c, 2); + + case 2: + if (s.len < 3) { + break; + } + c = wuffs_base__load_u24le__no_bounds_check(s.ptr); + if ((c & 0xC0C000) != 0x808000) { + break; + } + c = (0x00F000 & (c << 12)) | (0x000FC0 & (c >> 2)) | + (0x00003F & (c >> 16)); + if ((c <= 0x07FF) || ((0xD800 <= c) && (c <= 0xDFFF))) { + break; + } + return wuffs_base__make_utf_8__next__output(c, 3); + + case 3: + if (s.len < 4) { + break; + } + c = wuffs_base__load_u32le__no_bounds_check(s.ptr); + if ((c & 0xC0C0C000) != 0x80808000) { + break; + } + c = (0x1C0000 & (c << 18)) | (0x03F000 & (c << 4)) | + (0x000FC0 & (c >> 10)) | (0x00003F & (c >> 24)); + if ((c <= 0xFFFF) || (0x110000 <= c)) { + break; + } + return wuffs_base__make_utf_8__next__output(c, 4); + } + + return wuffs_base__make_utf_8__next__output( + WUFFS_BASE__UNICODE_REPLACEMENT_CHARACTER, 1); +} + +size_t // +wuffs_base__utf_8__longest_valid_prefix(wuffs_base__slice_u8 s) { + // TODO: possibly optimize the all-ASCII case (4 or 8 bytes at a time). + // + // TODO: possibly optimize this by manually inlining the + // wuffs_base__utf_8__next calls. + size_t original_len = s.len; + while (s.len > 0) { + wuffs_base__utf_8__next__output o = wuffs_base__utf_8__next(s); + if ((o.code_point > 0x7F) && (o.byte_length == 1)) { + break; + } + s.ptr += o.byte_length; + s.len -= o.byte_length; + } + return original_len - s.len; +} + +size_t // +wuffs_base__ascii__longest_valid_prefix(wuffs_base__slice_u8 s) { + // TODO: possibly optimize this by checking 4 or 8 bytes at a time. + uint8_t* original_ptr = s.ptr; + uint8_t* p = s.ptr; + uint8_t* q = s.ptr + s.len; + for (; (p != q) && ((*p & 0x80) == 0); p++) { + } + return (size_t)(p - original_ptr); +} diff --git a/internal/cgen/base/strconv-private.h b/internal/cgen/base/strconv-private.h index e9e2557c..4d255402 100644 --- a/internal/cgen/base/strconv-private.h +++ b/internal/cgen/base/strconv-private.h @@ -15,3 +15,5 @@ // limitations under the License. // ---------------- String Conversions + +// ---------------- Unicode and UTF-8 diff --git a/internal/cgen/base/strconv-public.h b/internal/cgen/base/strconv-public.h index caec8945..d4e394d4 100644 --- a/internal/cgen/base/strconv-public.h +++ b/internal/cgen/base/strconv-public.h @@ -52,3 +52,126 @@ wuffs_base__parse_number_i64(wuffs_base__slice_u8 s); // "__0D_1_002" would successfully parse as "one thousand and two". wuffs_base__result_u64 // wuffs_base__parse_number_u64(wuffs_base__slice_u8 s); + +// ---------------- Unicode and UTF-8 + +#define WUFFS_BASE__UNICODE_CODE_POINT__MIN_INCL 0x00000000 +#define WUFFS_BASE__UNICODE_CODE_POINT__MAX_INCL 0x0010FFFF + +#define WUFFS_BASE__UNICODE_REPLACEMENT_CHARACTER 0x0000FFFD + +#define WUFFS_BASE__UNICODE_SURROGATE__MIN_INCL 0x0000D800 +#define WUFFS_BASE__UNICODE_SURROGATE__MAX_INCL 0x0000DFFF + +#define WUFFS_BASE__ASCII__MIN_INCL 0x00 +#define WUFFS_BASE__ASCII__MAX_INCL 0x7F + +#define WUFFS_BASE__UTF_8__BYTE_LENGTH__MIN_INCL 1 +#define WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL 4 + +#define WUFFS_BASE__UTF_8__BYTE_LENGTH_1__CODE_POINT__MIN_INCL 0x00000000 +#define WUFFS_BASE__UTF_8__BYTE_LENGTH_1__CODE_POINT__MAX_INCL 0x0000007F +#define WUFFS_BASE__UTF_8__BYTE_LENGTH_2__CODE_POINT__MIN_INCL 0x00000080 +#define WUFFS_BASE__UTF_8__BYTE_LENGTH_2__CODE_POINT__MAX_INCL 0x000007FF +#define WUFFS_BASE__UTF_8__BYTE_LENGTH_3__CODE_POINT__MIN_INCL 0x00000800 +#define WUFFS_BASE__UTF_8__BYTE_LENGTH_3__CODE_POINT__MAX_INCL 0x0000FFFF +#define WUFFS_BASE__UTF_8__BYTE_LENGTH_4__CODE_POINT__MIN_INCL 0x00010000 +#define WUFFS_BASE__UTF_8__BYTE_LENGTH_4__CODE_POINT__MAX_INCL 0x0010FFFF + +// -------- + +// wuffs_base__utf_8__next__output is the type returned by +// wuffs_base__utf_8__next. +typedef struct { + uint32_t code_point; + uint32_t byte_length; + +#ifdef __cplusplus + inline bool is_valid() const; +#endif // __cplusplus + +} wuffs_base__utf_8__next__output; + +static inline wuffs_base__utf_8__next__output // +wuffs_base__make_utf_8__next__output(uint32_t code_point, + uint32_t byte_length) { + wuffs_base__utf_8__next__output ret; + ret.code_point = code_point; + ret.byte_length = byte_length; + return ret; +} + +static inline bool // +wuffs_base__utf_8__next__output__is_valid( + const wuffs_base__utf_8__next__output* o) { + if (o) { + uint32_t cp = o->code_point; + switch (o->byte_length) { + case 1: + return (cp <= 0x7F); + case 2: + return (0x080 <= cp) && (cp <= 0x7FF); + case 3: + // Avoid the 0xD800 ..= 0xDFFF surrogate range. + return ((0x0800 <= cp) && (cp <= 0xD7FF)) || + ((0xE000 <= cp) && (cp <= 0xFFFF)); + case 4: + return (0x00010000 <= cp) && (cp <= 0x0010FFFF); + } + } + return false; +} + +#ifdef __cplusplus + +inline bool // +wuffs_base__utf_8__next__output::is_valid() const { + return wuffs_base__utf_8__next__output__is_valid(this); +} + +#endif // __cplusplus + +// -------- + +// wuffs_base__utf_8__next returns the next UTF-8 code point (and that code +// point's byte length) at the start of s. +// +// There are exactly two cases in which this function returns something where +// wuffs_base__utf_8__next__output__is_valid is false: +// - If s is empty then it returns {.code_point=0, .byte_length=0}. +// - If s is non-empty and starts with invalid UTF-8 then it returns +// {.code_point=WUFFS_BASE__UNICODE_REPLACEMENT_CHARACTER, .byte_length=1}. +// +// Otherwise, it returns something where +// wuffs_base__utf_8__next__output__is_valid is true. +// +// In any case, it always returns an output that satisfies both of: +// - (output.code_point <= WUFFS_BASE__UNICODE_CODE_POINT__MAX_INCL). +// - (output.byte_length <= s.len). +// +// If s is a sub-slice of a larger slice of valid UTF-8, but that sub-slice +// boundary occurs in the middle of a multi-byte UTF-8 encoding of a single +// code point, then this function may return something invalid. It is the +// caller's responsibility to split on or otherwise manage UTF-8 boundaries. +wuffs_base__utf_8__next__output // +wuffs_base__utf_8__next(wuffs_base__slice_u8 s); + +// wuffs_base__utf_8__longest_valid_prefix returns the largest n such that the +// sub-slice s[..n] is valid UTF-8. +// +// In particular, it returns s.len if and only if all of s is valid UTF-8. +// +// If s is a sub-slice of a larger slice of valid UTF-8, but that sub-slice +// boundary occurs in the middle of a multi-byte UTF-8 encoding of a single +// code point, then this function will return less than s.len. It is the +// caller's responsibility to split on or otherwise manage UTF-8 boundaries. +size_t // +wuffs_base__utf_8__longest_valid_prefix(wuffs_base__slice_u8 s); + +// wuffs_base__ascii__longest_valid_prefix returns the largest n such that the +// sub-slice s[..n] is valid ASCII. +// +// In particular, it returns s.len if and only if all of s is valid ASCII. +// Equivalently, when none of the bytes in s have the 0x80 high bit set. +size_t // +wuffs_base__ascii__longest_valid_prefix(wuffs_base__slice_u8 s); diff --git a/internal/cgen/data.go b/internal/cgen/data.go index 885cfe04..ebb7fa9a 100644 --- a/internal/cgen/data.go +++ b/internal/cgen/data.go @@ -77,7 +77,14 @@ const baseStrConvImplC = "" + "alue = +(int64_t)(r.value);\n return ret;\n }\n } while (0);\n\nfail_bad_argument:\n do {\n wuffs_base__result_i64 ret;\n ret.status.repr = wuffs_base__error__bad_argument;\n ret.value = 0;\n return ret;\n } while (0);\n\nfail_out_of_bounds:\n do {\n wuffs_base__result_i64 ret;\n ret.status.repr = wuffs_base__error__out_of_bounds;\n ret.value = 0;\n return ret;\n } while (0);\n}\n\nwuffs_base__result_u64 //\nwuffs_base__parse_number_u64(wuffs_base__slice_u8 s) {\n uint8_t* p = s.ptr;\n uint8_t* q = s.ptr + s.len;\n\n for (; (p < q) && (*p == '_'); p++) {\n }\n\n if (p >= q) {\n goto fail_bad_argument;\n\n } else if (*p == '0') {\n p++;\n if (p >= q) {\n goto ok_zero;\n }\n if (*p == '_') {\n p++;\n for (; p < q; p++) {\n if (*p != '_') {\n goto fail_bad_argument;\n }\n }\n goto ok_zero;\n }\n\n if ((*p == 'x') || (*p == 'X')) {\n p++;\n for (; (p < q) && (*p == '_'); p++) {\n }\n if (p < q) {\n goto hexadecimal;\n }\n\n " + " } else if ((*p == 'd') || (*p == 'D')) {\n p++;\n for (; (p < q) && (*p == '_'); p++) {\n }\n if (p < q) {\n goto decimal;\n }\n }\n\n goto fail_bad_argument;\n }\n\ndecimal:\n do {\n uint64_t v = wuffs_base__parse_number__decimal_digits[*p++];\n if (v == 0) {\n goto fail_bad_argument;\n }\n v &= 0x0F;\n\n // UINT64_MAX is 18446744073709551615, which is ((10 * max10) + max1).\n const uint64_t max10 = 1844674407370955161;\n const uint8_t max1 = 5;\n\n for (; p < q; p++) {\n if (*p == '_') {\n continue;\n }\n uint8_t digit = wuffs_base__parse_number__decimal_digits[*p];\n if (digit == 0) {\n goto fail_bad_argument;\n }\n digit &= 0x0F;\n if ((v > max10) || ((v == max10) && (digit > max1))) {\n goto fail_out_of_bounds;\n }\n v = (10 * v) + ((uint64_t)(digit));\n }\n\n wuffs_base__result_u64 ret;\n ret.status.repr = NULL;\n ret.value = v;\n return ret;\n } while (0);\n\nhexadecimal:\n do {\n uint64_t v = " + "wuffs_base__parse_number__hexadecimal_digits[*p++];\n if (v == 0) {\n goto fail_bad_argument;\n }\n v &= 0x0F;\n\n for (; p < q; p++) {\n if (*p == '_') {\n continue;\n }\n uint8_t digit = wuffs_base__parse_number__hexadecimal_digits[*p];\n if (digit == 0) {\n goto fail_bad_argument;\n }\n digit &= 0x0F;\n if ((v >> 60) != 0) {\n goto fail_out_of_bounds;\n }\n v = (v << 4) | ((uint64_t)(digit));\n }\n\n wuffs_base__result_u64 ret;\n ret.status.repr = NULL;\n ret.value = v;\n return ret;\n } while (0);\n\nok_zero:\n do {\n wuffs_base__result_u64 ret;\n ret.status.repr = NULL;\n ret.value = 0;\n return ret;\n } while (0);\n\nfail_bad_argument:\n do {\n wuffs_base__result_u64 ret;\n ret.status.repr = wuffs_base__error__bad_argument;\n ret.value = 0;\n return ret;\n } while (0);\n\nfail_out_of_bounds:\n do {\n wuffs_base__result_u64 ret;\n ret.status.repr = wuffs_base__error__out_of_bounds;\n ret.value = 0;\n return ret;\n " + - " } while (0);\n}\n" + + " } while (0);\n}\n\n" + + "" + + "// ---------------- Unicode and UTF-8\n\n// wuffs_base__utf_8__byte_length_minus_1 is the byte length (minus 1) of a\n// UTF-8 encoded code point, based on the encoding's initial byte.\n// - 0x00 is 1-byte UTF-8 (ASCII).\n// - 0x01 is the start of 2-byte UTF-8.\n// - 0x02 is the start of 3-byte UTF-8.\n// - 0x03 is the start of 4-byte UTF-8.\n// - 0x40 is a UTF-8 tail byte.\n// - 0x80 is invalid UTF-8.\n//\n// RFC 3629 (UTF-8) gives this grammar for valid UTF-8:\n// UTF8-1 = %x00-7F\n// UTF8-2 = %xC2-DF UTF8-tail\n// UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) /\n// %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail )\n// UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) /\n// %xF4 %x80-8F 2( UTF8-tail )\n// UTF8-tail = %x80-BF\nstatic const uint8_t wuffs_base__utf_8__byte_length_minus_1[256] = {\n // 0 1 2 3 4 5 6 7\n // 8 9 A B C D E F\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x" + + "00, 0x00, 0x00, // 0x00 ..= 0x07.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x08 ..= 0x0F.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x10 ..= 0x17.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x18 ..= 0x1F.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20 ..= 0x27.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x28 ..= 0x2F.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x30 ..= 0x37.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x38 ..= 0x3F.\n\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x40 ..= 0x47.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x48 ..= 0x4F.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x50 ..= 0x57.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x58 ..= 0x5F.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x60 ..= 0x67.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x68 ..= 0x6F.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x70 ..= " + + "0x77.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x78 ..= 0x7F.\n\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x80 ..= 0x87.\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x88 ..= 0x8F.\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x90 ..= 0x97.\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x98 ..= 0x9F.\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xA0 ..= 0xA7.\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xA8 ..= 0xAF.\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xB0 ..= 0xB7.\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xB8 ..= 0xBF.\n\n 0x80, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xC0 ..= 0xC7.\n 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xC8 ..= 0xCF.\n 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xD0 ..= 0xD7.\n 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xD8 ..= 0xDF.\n 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, // 0xE0 ..= 0xE7.\n 0x02, 0x02, 0x02, " + + "0x02, 0x02, 0x02, 0x02, 0x02, // 0xE8 ..= 0xEF.\n 0x03, 0x03, 0x03, 0x03, 0x03, 0x80, 0x80, 0x80, // 0xF0 ..= 0xF7.\n 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xF8 ..= 0xFF.\n // 0 1 2 3 4 5 6 7\n // 8 9 A B C D E F\n};\n\nwuffs_base__utf_8__next__output //\nwuffs_base__utf_8__next(wuffs_base__slice_u8 s) {\n if (s.len == 0) {\n return wuffs_base__make_utf_8__next__output(0, 0);\n }\n uint32_t c = s.ptr[0];\n switch (wuffs_base__utf_8__byte_length_minus_1[c & 0xFF]) {\n case 0:\n return wuffs_base__make_utf_8__next__output(c, 1);\n\n case 1:\n if (s.len < 2) {\n break;\n }\n c = wuffs_base__load_u16le__no_bounds_check(s.ptr);\n if ((c & 0xC000) != 0x8000) {\n break;\n }\n c = (0x0007C0 & (c << 6)) | (0x00003F & (c >> 8));\n return wuffs_base__make_utf_8__next__output(c, 2);\n\n case 2:\n if (s.len < 3) {\n break;\n }\n c = wuffs_base__load_u24le__no_bounds_check(s.ptr);" + + "\n if ((c & 0xC0C000) != 0x808000) {\n break;\n }\n c = (0x00F000 & (c << 12)) | (0x000FC0 & (c >> 2)) |\n (0x00003F & (c >> 16));\n if ((c <= 0x07FF) || ((0xD800 <= c) && (c <= 0xDFFF))) {\n break;\n }\n return wuffs_base__make_utf_8__next__output(c, 3);\n\n case 3:\n if (s.len < 4) {\n break;\n }\n c = wuffs_base__load_u32le__no_bounds_check(s.ptr);\n if ((c & 0xC0C0C000) != 0x80808000) {\n break;\n }\n c = (0x1C0000 & (c << 18)) | (0x03F000 & (c << 4)) |\n (0x000FC0 & (c >> 10)) | (0x00003F & (c >> 24));\n if ((c <= 0xFFFF) || (0x110000 <= c)) {\n break;\n }\n return wuffs_base__make_utf_8__next__output(c, 4);\n }\n\n return wuffs_base__make_utf_8__next__output(\n WUFFS_BASE__UNICODE_REPLACEMENT_CHARACTER, 1);\n}\n\nsize_t //\nwuffs_base__utf_8__longest_valid_prefix(wuffs_base__slice_u8 s) {\n // TODO: possibly optimize the all-ASCII case (4 or 8 bytes at a time).\n //\n // TODO: possibly optimize thi" + + "s by manually inlining the\n // wuffs_base__utf_8__next calls.\n size_t original_len = s.len;\n while (s.len > 0) {\n wuffs_base__utf_8__next__output o = wuffs_base__utf_8__next(s);\n if ((o.code_point > 0x7F) && (o.byte_length == 1)) {\n break;\n }\n s.ptr += o.byte_length;\n s.len -= o.byte_length;\n }\n return original_len - s.len;\n}\n\nsize_t //\nwuffs_base__ascii__longest_valid_prefix(wuffs_base__slice_u8 s) {\n // TODO: possibly optimize this by checking 4 or 8 bytes at a time.\n uint8_t* original_ptr = s.ptr;\n uint8_t* p = s.ptr;\n uint8_t* q = s.ptr + s.len;\n for (; (p != q) && ((*p & 0x80) == 0); p++) {\n }\n return (size_t)(p - original_ptr);\n}\n" + "" const baseCorePrivateH = "" + @@ -273,13 +280,25 @@ const baseRangePublicH = "" + "" const baseStrConvPrivateH = "" + - "// ---------------- String Conversions\n" + + "// ---------------- String Conversions\n\n" + + "" + + "// ---------------- Unicode and UTF-8\n" + "" const baseStrConvPublicH = "" + "// ---------------- String Conversions\n\n// wuffs_base__parse_number_i64 parses the ASCII integer in s. For example, if\n// s contains the bytes \"-123\" then it will return the int64_t -123.\n//\n// It returns an error if s does not contain an integer or if the integer\n// within would overflow an int64_t.\n//\n// It is similar to wuffs_base__parse_number_u64 but it returns a signed\n// integer, not an unsigned integer. It also allows a leading '+' or '-'.\nwuffs_base__result_i64 //\nwuffs_base__parse_number_i64(wuffs_base__slice_u8 s);\n\n// wuffs_base__parse_number_u64 parses the ASCII integer in s. For example, if\n// s contains the bytes \"123\" then it will return the uint64_t 123.\n//\n// It returns an error if s does not contain an integer or if the integer\n// within would overflow a uint64_t.\n//\n// It is similar to the C standard library's strtoull function, but:\n// - Errors are returned in-band (in a result type), not out-of-band (errno).\n// - It takes a slice (a pointer and length), not a NUL-terminated C string.\n" + "// - It does not take an optional endptr argument. It does not allow a partial\n// parse: it returns an error unless all of s is consumed.\n// - It does not allow whitespace, leading or otherwise.\n// - It does not allow a leading '+' or '-'.\n// - It does not allow multiple leading zeroes: \"0\" is valid but \"00\" is not.\n// - It does not take a base argument (e.g. base 10 vs base 16). Instead, it\n// always accepts both decimal (e.g \"1234\", \"0d5678\") and hexadecimal (e.g.\n// \"0x9aBC\"). The caller is responsible for prior filtering of e.g. hex\n// numbers if they are unwanted. For example, Wuffs' JSON decoder will only\n// produce a wuffs_base__token for decimal numbers, not hexadecimal.\n// - It is not affected by i18n / l10n settings.\n// - It does allow arbitrary underscores, except inside the optional 2-byte\n// opening \"0d\" or \"0X\" that denotes base-10 or base-16. For example,\n// \"__0D_1_002\" would successfully parse as \"one thousand and two\".\nwuffs_base__result_u64 //\nwuffs_base__parse_" + - "number_u64(wuffs_base__slice_u8 s);\n" + + "number_u64(wuffs_base__slice_u8 s);\n\n" + + "" + + "// ---------------- Unicode and UTF-8\n\n#define WUFFS_BASE__UNICODE_CODE_POINT__MIN_INCL 0x00000000\n#define WUFFS_BASE__UNICODE_CODE_POINT__MAX_INCL 0x0010FFFF\n\n#define WUFFS_BASE__UNICODE_REPLACEMENT_CHARACTER 0x0000FFFD\n\n#define WUFFS_BASE__UNICODE_SURROGATE__MIN_INCL 0x0000D800\n#define WUFFS_BASE__UNICODE_SURROGATE__MAX_INCL 0x0000DFFF\n\n#define WUFFS_BASE__ASCII__MIN_INCL 0x00\n#define WUFFS_BASE__ASCII__MAX_INCL 0x7F\n\n#define WUFFS_BASE__UTF_8__BYTE_LENGTH__MIN_INCL 1\n#define WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL 4\n\n#define WUFFS_BASE__UTF_8__BYTE_LENGTH_1__CODE_POINT__MIN_INCL 0x00000000\n#define WUFFS_BASE__UTF_8__BYTE_LENGTH_1__CODE_POINT__MAX_INCL 0x0000007F\n#define WUFFS_BASE__UTF_8__BYTE_LENGTH_2__CODE_POINT__MIN_INCL 0x00000080\n#define WUFFS_BASE__UTF_8__BYTE_LENGTH_2__CODE_POINT__MAX_INCL 0x000007FF\n#define WUFFS_BASE__UTF_8__BYTE_LENGTH_3__CODE_POINT__MIN_INCL 0x00000800\n#define WUFFS_BASE__UTF_8__BYTE_LENGTH_3__CODE_POINT__MAX_INCL 0x0000FFFF\n#define WUFFS_BASE__UTF_8__BYTE_LENGTH_4__CODE_POINT_" + + "_MIN_INCL 0x00010000\n#define WUFFS_BASE__UTF_8__BYTE_LENGTH_4__CODE_POINT__MAX_INCL 0x0010FFFF\n\n" + + "" + + "// --------\n\n// wuffs_base__utf_8__next__output is the type returned by\n// wuffs_base__utf_8__next.\ntypedef struct {\n uint32_t code_point;\n uint32_t byte_length;\n\n#ifdef __cplusplus\n inline bool is_valid() const;\n#endif // __cplusplus\n\n} wuffs_base__utf_8__next__output;\n\nstatic inline wuffs_base__utf_8__next__output //\nwuffs_base__make_utf_8__next__output(uint32_t code_point,\n uint32_t byte_length) {\n wuffs_base__utf_8__next__output ret;\n ret.code_point = code_point;\n ret.byte_length = byte_length;\n return ret;\n}\n\nstatic inline bool //\nwuffs_base__utf_8__next__output__is_valid(\n const wuffs_base__utf_8__next__output* o) {\n if (o) {\n uint32_t cp = o->code_point;\n switch (o->byte_length) {\n case 1:\n return (cp <= 0x7F);\n case 2:\n return (0x080 <= cp) && (cp <= 0x7FF);\n case 3:\n // Avoid the 0xD800 ..= 0xDFFF surrogate range.\n return ((0x0800 <= cp) && (cp <= 0xD7FF)) ||\n ((0xE000 <= cp) && (cp <= 0xF" + + "FFF));\n case 4:\n return (0x00010000 <= cp) && (cp <= 0x0010FFFF);\n }\n }\n return false;\n}\n\n#ifdef __cplusplus\n\ninline bool //\nwuffs_base__utf_8__next__output::is_valid() const {\n return wuffs_base__utf_8__next__output__is_valid(this);\n}\n\n#endif // __cplusplus\n\n" + + "" + + "// --------\n\n// wuffs_base__utf_8__next returns the next UTF-8 code point (and that code\n// point's byte length) at the start of s.\n//\n// There are exactly two cases in which this function returns something where\n// wuffs_base__utf_8__next__output__is_valid is false:\n// - If s is empty then it returns {.code_point=0, .byte_length=0}.\n// - If s is non-empty and starts with invalid UTF-8 then it returns\n// {.code_point=WUFFS_BASE__UNICODE_REPLACEMENT_CHARACTER, .byte_length=1}.\n//\n// Otherwise, it returns something where\n// wuffs_base__utf_8__next__output__is_valid is true.\n//\n// In any case, it always returns an output that satisfies both of:\n// - (output.code_point <= WUFFS_BASE__UNICODE_CODE_POINT__MAX_INCL).\n// - (output.byte_length <= s.len).\n//\n// If s is a sub-slice of a larger slice of valid UTF-8, but that sub-slice\n// boundary occurs in the middle of a multi-byte UTF-8 encoding of a single\n// code point, then this function may return something invalid. It is the\n// caller's responsibility to s" + + "plit on or otherwise manage UTF-8 boundaries.\nwuffs_base__utf_8__next__output //\nwuffs_base__utf_8__next(wuffs_base__slice_u8 s);\n\n// wuffs_base__utf_8__longest_valid_prefix returns the largest n such that the\n// sub-slice s[..n] is valid UTF-8.\n//\n// In particular, it returns s.len if and only if all of s is valid UTF-8.\n//\n// If s is a sub-slice of a larger slice of valid UTF-8, but that sub-slice\n// boundary occurs in the middle of a multi-byte UTF-8 encoding of a single\n// code point, then this function will return less than s.len. It is the\n// caller's responsibility to split on or otherwise manage UTF-8 boundaries.\nsize_t //\nwuffs_base__utf_8__longest_valid_prefix(wuffs_base__slice_u8 s);\n\n// wuffs_base__ascii__longest_valid_prefix returns the largest n such that the\n// sub-slice s[..n] is valid ASCII.\n//\n// In particular, it returns s.len if and only if all of s is valid ASCII.\n// Equivalently, when none of the bytes in s have the 0x80 high bit set.\nsize_t //\nwuffs_base__ascii__longest_valid_prefix(" + + "wuffs_base__slice_u8 s);\n" + "" const baseTokenPrivateH = "" + diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 3ae5c6a7..6b3a64a4 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -3401,6 +3401,129 @@ wuffs_base__parse_number_i64(wuffs_base__slice_u8 s); wuffs_base__result_u64 // wuffs_base__parse_number_u64(wuffs_base__slice_u8 s); +// ---------------- Unicode and UTF-8 + +#define WUFFS_BASE__UNICODE_CODE_POINT__MIN_INCL 0x00000000 +#define WUFFS_BASE__UNICODE_CODE_POINT__MAX_INCL 0x0010FFFF + +#define WUFFS_BASE__UNICODE_REPLACEMENT_CHARACTER 0x0000FFFD + +#define WUFFS_BASE__UNICODE_SURROGATE__MIN_INCL 0x0000D800 +#define WUFFS_BASE__UNICODE_SURROGATE__MAX_INCL 0x0000DFFF + +#define WUFFS_BASE__ASCII__MIN_INCL 0x00 +#define WUFFS_BASE__ASCII__MAX_INCL 0x7F + +#define WUFFS_BASE__UTF_8__BYTE_LENGTH__MIN_INCL 1 +#define WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL 4 + +#define WUFFS_BASE__UTF_8__BYTE_LENGTH_1__CODE_POINT__MIN_INCL 0x00000000 +#define WUFFS_BASE__UTF_8__BYTE_LENGTH_1__CODE_POINT__MAX_INCL 0x0000007F +#define WUFFS_BASE__UTF_8__BYTE_LENGTH_2__CODE_POINT__MIN_INCL 0x00000080 +#define WUFFS_BASE__UTF_8__BYTE_LENGTH_2__CODE_POINT__MAX_INCL 0x000007FF +#define WUFFS_BASE__UTF_8__BYTE_LENGTH_3__CODE_POINT__MIN_INCL 0x00000800 +#define WUFFS_BASE__UTF_8__BYTE_LENGTH_3__CODE_POINT__MAX_INCL 0x0000FFFF +#define WUFFS_BASE__UTF_8__BYTE_LENGTH_4__CODE_POINT__MIN_INCL 0x00010000 +#define WUFFS_BASE__UTF_8__BYTE_LENGTH_4__CODE_POINT__MAX_INCL 0x0010FFFF + +// -------- + +// wuffs_base__utf_8__next__output is the type returned by +// wuffs_base__utf_8__next. +typedef struct { + uint32_t code_point; + uint32_t byte_length; + +#ifdef __cplusplus + inline bool is_valid() const; +#endif // __cplusplus + +} wuffs_base__utf_8__next__output; + +static inline wuffs_base__utf_8__next__output // +wuffs_base__make_utf_8__next__output(uint32_t code_point, + uint32_t byte_length) { + wuffs_base__utf_8__next__output ret; + ret.code_point = code_point; + ret.byte_length = byte_length; + return ret; +} + +static inline bool // +wuffs_base__utf_8__next__output__is_valid( + const wuffs_base__utf_8__next__output* o) { + if (o) { + uint32_t cp = o->code_point; + switch (o->byte_length) { + case 1: + return (cp <= 0x7F); + case 2: + return (0x080 <= cp) && (cp <= 0x7FF); + case 3: + // Avoid the 0xD800 ..= 0xDFFF surrogate range. + return ((0x0800 <= cp) && (cp <= 0xD7FF)) || + ((0xE000 <= cp) && (cp <= 0xFFFF)); + case 4: + return (0x00010000 <= cp) && (cp <= 0x0010FFFF); + } + } + return false; +} + +#ifdef __cplusplus + +inline bool // +wuffs_base__utf_8__next__output::is_valid() const { + return wuffs_base__utf_8__next__output__is_valid(this); +} + +#endif // __cplusplus + +// -------- + +// wuffs_base__utf_8__next returns the next UTF-8 code point (and that code +// point's byte length) at the start of s. +// +// There are exactly two cases in which this function returns something where +// wuffs_base__utf_8__next__output__is_valid is false: +// - If s is empty then it returns {.code_point=0, .byte_length=0}. +// - If s is non-empty and starts with invalid UTF-8 then it returns +// {.code_point=WUFFS_BASE__UNICODE_REPLACEMENT_CHARACTER, .byte_length=1}. +// +// Otherwise, it returns something where +// wuffs_base__utf_8__next__output__is_valid is true. +// +// In any case, it always returns an output that satisfies both of: +// - (output.code_point <= WUFFS_BASE__UNICODE_CODE_POINT__MAX_INCL). +// - (output.byte_length <= s.len). +// +// If s is a sub-slice of a larger slice of valid UTF-8, but that sub-slice +// boundary occurs in the middle of a multi-byte UTF-8 encoding of a single +// code point, then this function may return something invalid. It is the +// caller's responsibility to split on or otherwise manage UTF-8 boundaries. +wuffs_base__utf_8__next__output // +wuffs_base__utf_8__next(wuffs_base__slice_u8 s); + +// wuffs_base__utf_8__longest_valid_prefix returns the largest n such that the +// sub-slice s[..n] is valid UTF-8. +// +// In particular, it returns s.len if and only if all of s is valid UTF-8. +// +// If s is a sub-slice of a larger slice of valid UTF-8, but that sub-slice +// boundary occurs in the middle of a multi-byte UTF-8 encoding of a single +// code point, then this function will return less than s.len. It is the +// caller's responsibility to split on or otherwise manage UTF-8 boundaries. +size_t // +wuffs_base__utf_8__longest_valid_prefix(wuffs_base__slice_u8 s); + +// wuffs_base__ascii__longest_valid_prefix returns the largest n such that the +// sub-slice s[..n] is valid ASCII. +// +// In particular, it returns s.len if and only if all of s is valid ASCII. +// Equivalently, when none of the bytes in s have the 0x80 high bit set. +size_t // +wuffs_base__ascii__longest_valid_prefix(wuffs_base__slice_u8 s); + // ---------------- Interface Declarations. extern const char* wuffs_base__hasher_u32__vtable_name; @@ -6633,6 +6756,8 @@ wuffs_base__io_writer__set(wuffs_base__io_buffer* b, // ---------------- String Conversions + // ---------------- Unicode and UTF-8 + // ---------------- #if !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__BASE) @@ -8448,6 +8573,152 @@ fail_out_of_bounds: } while (0); } +// ---------------- Unicode and UTF-8 + +// wuffs_base__utf_8__byte_length_minus_1 is the byte length (minus 1) of a +// UTF-8 encoded code point, based on the encoding's initial byte. +// - 0x00 is 1-byte UTF-8 (ASCII). +// - 0x01 is the start of 2-byte UTF-8. +// - 0x02 is the start of 3-byte UTF-8. +// - 0x03 is the start of 4-byte UTF-8. +// - 0x40 is a UTF-8 tail byte. +// - 0x80 is invalid UTF-8. +// +// RFC 3629 (UTF-8) gives this grammar for valid UTF-8: +// UTF8-1 = %x00-7F +// UTF8-2 = %xC2-DF UTF8-tail +// UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / +// %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) +// UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / +// %xF4 %x80-8F 2( UTF8-tail ) +// UTF8-tail = %x80-BF +static const uint8_t wuffs_base__utf_8__byte_length_minus_1[256] = { + // 0 1 2 3 4 5 6 7 + // 8 9 A B C D E F + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x00 ..= 0x07. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x08 ..= 0x0F. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x10 ..= 0x17. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x18 ..= 0x1F. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20 ..= 0x27. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x28 ..= 0x2F. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x30 ..= 0x37. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x38 ..= 0x3F. + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x40 ..= 0x47. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x48 ..= 0x4F. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x50 ..= 0x57. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x58 ..= 0x5F. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x60 ..= 0x67. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x68 ..= 0x6F. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x70 ..= 0x77. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x78 ..= 0x7F. + + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x80 ..= 0x87. + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x88 ..= 0x8F. + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x90 ..= 0x97. + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x98 ..= 0x9F. + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xA0 ..= 0xA7. + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xA8 ..= 0xAF. + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xB0 ..= 0xB7. + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xB8 ..= 0xBF. + + 0x80, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xC0 ..= 0xC7. + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xC8 ..= 0xCF. + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xD0 ..= 0xD7. + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xD8 ..= 0xDF. + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, // 0xE0 ..= 0xE7. + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, // 0xE8 ..= 0xEF. + 0x03, 0x03, 0x03, 0x03, 0x03, 0x80, 0x80, 0x80, // 0xF0 ..= 0xF7. + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xF8 ..= 0xFF. + // 0 1 2 3 4 5 6 7 + // 8 9 A B C D E F +}; + +wuffs_base__utf_8__next__output // +wuffs_base__utf_8__next(wuffs_base__slice_u8 s) { + if (s.len == 0) { + return wuffs_base__make_utf_8__next__output(0, 0); + } + uint32_t c = s.ptr[0]; + switch (wuffs_base__utf_8__byte_length_minus_1[c & 0xFF]) { + case 0: + return wuffs_base__make_utf_8__next__output(c, 1); + + case 1: + if (s.len < 2) { + break; + } + c = wuffs_base__load_u16le__no_bounds_check(s.ptr); + if ((c & 0xC000) != 0x8000) { + break; + } + c = (0x0007C0 & (c << 6)) | (0x00003F & (c >> 8)); + return wuffs_base__make_utf_8__next__output(c, 2); + + case 2: + if (s.len < 3) { + break; + } + c = wuffs_base__load_u24le__no_bounds_check(s.ptr); + if ((c & 0xC0C000) != 0x808000) { + break; + } + c = (0x00F000 & (c << 12)) | (0x000FC0 & (c >> 2)) | + (0x00003F & (c >> 16)); + if ((c <= 0x07FF) || ((0xD800 <= c) && (c <= 0xDFFF))) { + break; + } + return wuffs_base__make_utf_8__next__output(c, 3); + + case 3: + if (s.len < 4) { + break; + } + c = wuffs_base__load_u32le__no_bounds_check(s.ptr); + if ((c & 0xC0C0C000) != 0x80808000) { + break; + } + c = (0x1C0000 & (c << 18)) | (0x03F000 & (c << 4)) | + (0x000FC0 & (c >> 10)) | (0x00003F & (c >> 24)); + if ((c <= 0xFFFF) || (0x110000 <= c)) { + break; + } + return wuffs_base__make_utf_8__next__output(c, 4); + } + + return wuffs_base__make_utf_8__next__output( + WUFFS_BASE__UNICODE_REPLACEMENT_CHARACTER, 1); +} + +size_t // +wuffs_base__utf_8__longest_valid_prefix(wuffs_base__slice_u8 s) { + // TODO: possibly optimize the all-ASCII case (4 or 8 bytes at a time). + // + // TODO: possibly optimize this by manually inlining the + // wuffs_base__utf_8__next calls. + size_t original_len = s.len; + while (s.len > 0) { + wuffs_base__utf_8__next__output o = wuffs_base__utf_8__next(s); + if ((o.code_point > 0x7F) && (o.byte_length == 1)) { + break; + } + s.ptr += o.byte_length; + s.len -= o.byte_length; + } + return original_len - s.len; +} + +size_t // +wuffs_base__ascii__longest_valid_prefix(wuffs_base__slice_u8 s) { + // TODO: possibly optimize this by checking 4 or 8 bytes at a time. + uint8_t* original_ptr = s.ptr; + uint8_t* p = s.ptr; + uint8_t* q = s.ptr + s.len; + for (; (p != q) && ((*p & 0x80) == 0); p++) { + } + return (size_t)(p - original_ptr); +} + #endif // !defined(WUFFS_CONFIG__MODULES) || // defined(WUFFS_CONFIG__MODULE__BASE) diff --git a/test/c/std/json.c b/test/c/std/json.c index 64a5cfb5..48db9833 100644 --- a/test/c/std/json.c +++ b/test/c/std/json.c @@ -198,6 +198,115 @@ test_strconv_parse_number_u64() { return NULL; } +const char* // +test_strconv_utf_8_next() { + CHECK_FOCUS(__func__); + + // Special case the "\x00" string, which is valid UTF-8 but its strlen is + // zero, not one. + uint8_t the_nul_byte[0]; + the_nul_byte[0] = '\x00'; + + struct { + uint32_t want_cp; + uint32_t want_bl; + const char* str; + } test_cases[] = { + {.want_cp = 0x00000000, .want_bl = 0, .str = ""}, + {.want_cp = 0x00000000, .want_bl = 1, .str = "The byte"}, + {.want_cp = 0x00000009, .want_bl = 1, .str = "\t"}, + {.want_cp = 0x00000041, .want_bl = 1, .str = "A"}, + {.want_cp = 0x00000061, .want_bl = 1, .str = "abdefghij"}, + {.want_cp = 0x0000007F, .want_bl = 1, .str = "\x7F"}, + {.want_cp = 0x00000080, .want_bl = 2, .str = "\xC2\x80"}, + {.want_cp = 0x000007FF, .want_bl = 2, .str = "\xDF\xBF"}, + {.want_cp = 0x00000800, .want_bl = 3, .str = "\xE0\xA0\x80"}, + {.want_cp = 0x0000FFFD, .want_bl = 3, .str = "\xEF\xBF\xBD"}, + {.want_cp = 0x0000FFFF, .want_bl = 3, .str = "\xEF\xBF\xBF"}, + {.want_cp = 0x00010000, .want_bl = 4, .str = "\xF0\x90\x80\x80"}, + {.want_cp = 0x0010FFFF, .want_bl = 4, .str = "\xF4\x8F\xBF\xBF"}, + + // U+00000394 GREEK CAPITAL LETTER DELTA. + {.want_cp = 0x00000394, .want_bl = 2, .str = "\xCE\x94"}, + {.want_cp = 0x00000394, .want_bl = 2, .str = "\xCE\x94+"}, + {.want_cp = 0x00000394, .want_bl = 2, .str = "\xCE\x94++"}, + {.want_cp = 0x00000394, .want_bl = 2, .str = "\xCE\x94+++"}, + {.want_cp = 0x00000394, .want_bl = 2, .str = "\xCE\x94++++"}, + {.want_cp = 0x00000394, .want_bl = 2, .str = "\xCE\x94\x80"}, + {.want_cp = 0x00000394, .want_bl = 2, .str = "\xCE\x94\x80\x80"}, + {.want_cp = 0x00000394, .want_bl = 2, .str = "\xCE\x94\x80\x80\x80"}, + {.want_cp = 0x00000394, .want_bl = 2, .str = "\xCE\x94\x80\x80\x80\x80"}, + + // U+00002603 SNOWMAN. + {.want_cp = 0x00002603, .want_bl = 3, .str = "\xE2\x98\x83"}, + {.want_cp = 0x00002603, .want_bl = 3, .str = "\xE2\x98\x83+"}, + {.want_cp = 0x00002603, .want_bl = 3, .str = "\xE2\x98\x83++"}, + {.want_cp = 0x00002603, .want_bl = 3, .str = "\xE2\x98\x83+++"}, + {.want_cp = 0x00002603, .want_bl = 3, .str = "\xE2\x98\x83++++"}, + {.want_cp = 0x00002603, .want_bl = 3, .str = "\xE2\x98\x83\xFF"}, + + // U+0001F4A9 PILE OF POO. + {.want_cp = 0x0001F4A9, .want_bl = 4, .str = "\xF0\x9F\x92\xA9"}, + {.want_cp = 0x0001F4A9, .want_bl = 4, .str = "\xF0\x9F\x92\xA9+"}, + {.want_cp = 0x0001F4A9, .want_bl = 4, .str = "\xF0\x9F\x92\xA9++"}, + {.want_cp = 0x0001F4A9, .want_bl = 4, .str = "\xF0\x9F\x92\xA9+++"}, + {.want_cp = 0x0001F4A9, .want_bl = 4, .str = "\xF0\x9F\x92\xA9++++"}, + {.want_cp = 0x0001F4A9, .want_bl = 4, .str = "\xF0\x9F\x92\xA9\xFF"}, + + // Invalid. + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\x80"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xBF"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xC0\x80"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xC1\xBF"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xC2"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xC2\x7F"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xC2\xC0"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xC2\xFF"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xCE"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xDF\xC0"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xDF\xFF"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xE0\x80"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xE0\x80\x80"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xE0\x9F\xBF"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xE2"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xF0"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xF0\x80\x80"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xF0\x80\x80\x80"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xF0\x8F\xBF\xBF"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xF4\x90\x80\x80"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xF5"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xF6\x80"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xF7\x80\x80"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xFF\xFF\xFF\xFF"}, + + // Invalid. UTF-8 cannot contain the surrogates U+D800 ..= U+DFFF. + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xED\xA0\x80"}, + {.want_cp = 0x0000FFFD, .want_bl = 1, .str = "\xED\xBF\xBF"}, + }; + + int tc; + for (tc = 0; tc < WUFFS_TESTLIB_ARRAY_SIZE(test_cases); tc++) { + wuffs_base__slice_u8 s = wuffs_base__make_slice_u8( + (void*)test_cases[tc].str, strlen(test_cases[tc].str)); + + // Override "The byte" with "\x00". + if ((test_cases[tc].want_cp == 0) && (test_cases[tc].want_bl == 1)) { + s = wuffs_base__make_slice_u8(&the_nul_byte[0], 1); + } + + wuffs_base__utf_8__next__output have = wuffs_base__utf_8__next(s); + if ((have.code_point != test_cases[tc].want_cp) || + (have.byte_length != test_cases[tc].want_bl)) { + RETURN_FAIL("\"%s\": have cp=0x%" PRIX32 " bl=%" PRIu32 + ", want cp=0x%" PRIX32 " bl=%" PRIu32, + test_cases[tc].str, have.code_point, have.byte_length, + test_cases[tc].want_cp, test_cases[tc].want_bl); + } + } + + return NULL; +} + // ---------------- Golden Tests golden_test json_australian_abc_gt = { @@ -562,6 +671,7 @@ proc tests[] = { // as any other place. test_strconv_parse_number_i64, // test_strconv_parse_number_u64, // + test_strconv_utf_8_next, // test_wuffs_json_decode_interface, // test_wuffs_json_decode_string, //