mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-21 16:47:28 +01:00
Add wuffs_base__utf_8__next
This commit is contained in:
1 parent
1ad5cfac4d
commit
809220e6cc
7 files changed
+676
-3
No files matched your search
@@ -3401,6 +3401,129 @@ wuffs_base__parse_number_i64(wuffs_base__slice_u8 s);
|
||||
wuffs_base__result_u64 //
|
||||
wuffs_base__parse_number_u64(wuffs_base__slice_u8 s);
|
||||
|
||||
// ---------------- Unicode and UTF-8
|
||||
|
||||
#define WUFFS_BASE__UNICODE_CODE_POINT__MIN_INCL 0x00000000
|
||||
#define WUFFS_BASE__UNICODE_CODE_POINT__MAX_INCL 0x0010FFFF
|
||||
|
||||
#define WUFFS_BASE__UNICODE_REPLACEMENT_CHARACTER 0x0000FFFD
|
||||
|
||||
#define WUFFS_BASE__UNICODE_SURROGATE__MIN_INCL 0x0000D800
|
||||
#define WUFFS_BASE__UNICODE_SURROGATE__MAX_INCL 0x0000DFFF
|
||||
|
||||
#define WUFFS_BASE__ASCII__MIN_INCL 0x00
|
||||
#define WUFFS_BASE__ASCII__MAX_INCL 0x7F
|
||||
|
||||
#define WUFFS_BASE__UTF_8__BYTE_LENGTH__MIN_INCL 1
|
||||
#define WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL 4
|
||||
|
||||
#define WUFFS_BASE__UTF_8__BYTE_LENGTH_1__CODE_POINT__MIN_INCL 0x00000000
|
||||
#define WUFFS_BASE__UTF_8__BYTE_LENGTH_1__CODE_POINT__MAX_INCL 0x0000007F
|
||||
#define WUFFS_BASE__UTF_8__BYTE_LENGTH_2__CODE_POINT__MIN_INCL 0x00000080
|
||||
#define WUFFS_BASE__UTF_8__BYTE_LENGTH_2__CODE_POINT__MAX_INCL 0x000007FF
|
||||
#define WUFFS_BASE__UTF_8__BYTE_LENGTH_3__CODE_POINT__MIN_INCL 0x00000800
|
||||
#define WUFFS_BASE__UTF_8__BYTE_LENGTH_3__CODE_POINT__MAX_INCL 0x0000FFFF
|
||||
#define WUFFS_BASE__UTF_8__BYTE_LENGTH_4__CODE_POINT__MIN_INCL 0x00010000
|
||||
#define WUFFS_BASE__UTF_8__BYTE_LENGTH_4__CODE_POINT__MAX_INCL 0x0010FFFF
|
||||
|
||||
// --------
|
||||
|
||||
// wuffs_base__utf_8__next__output is the type returned by
|
||||
// wuffs_base__utf_8__next.
|
||||
typedef struct {
|
||||
uint32_t code_point;
|
||||
uint32_t byte_length;
|
||||
|
||||
#ifdef __cplusplus
|
||||
inline bool is_valid() const;
|
||||
#endif // __cplusplus
|
||||
|
||||
} wuffs_base__utf_8__next__output;
|
||||
|
||||
static inline wuffs_base__utf_8__next__output //
|
||||
wuffs_base__make_utf_8__next__output(uint32_t code_point,
|
||||
uint32_t byte_length) {
|
||||
wuffs_base__utf_8__next__output ret;
|
||||
ret.code_point = code_point;
|
||||
ret.byte_length = byte_length;
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline bool //
|
||||
wuffs_base__utf_8__next__output__is_valid(
|
||||
const wuffs_base__utf_8__next__output* o) {
|
||||
if (o) {
|
||||
uint32_t cp = o->code_point;
|
||||
switch (o->byte_length) {
|
||||
case 1:
|
||||
return (cp <= 0x7F);
|
||||
case 2:
|
||||
return (0x080 <= cp) && (cp <= 0x7FF);
|
||||
case 3:
|
||||
// Avoid the 0xD800 ..= 0xDFFF surrogate range.
|
||||
return ((0x0800 <= cp) && (cp <= 0xD7FF)) ||
|
||||
((0xE000 <= cp) && (cp <= 0xFFFF));
|
||||
case 4:
|
||||
return (0x00010000 <= cp) && (cp <= 0x0010FFFF);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
inline bool //
|
||||
wuffs_base__utf_8__next__output::is_valid() const {
|
||||
return wuffs_base__utf_8__next__output__is_valid(this);
|
||||
}
|
||||
|
||||
#endif // __cplusplus
|
||||
|
||||
// --------
|
||||
|
||||
// wuffs_base__utf_8__next returns the next UTF-8 code point (and that code
|
||||
// point's byte length) at the start of s.
|
||||
//
|
||||
// There are exactly two cases in which this function returns something where
|
||||
// wuffs_base__utf_8__next__output__is_valid is false:
|
||||
// - If s is empty then it returns {.code_point=0, .byte_length=0}.
|
||||
// - If s is non-empty and starts with invalid UTF-8 then it returns
|
||||
// {.code_point=WUFFS_BASE__UNICODE_REPLACEMENT_CHARACTER, .byte_length=1}.
|
||||
//
|
||||
// Otherwise, it returns something where
|
||||
// wuffs_base__utf_8__next__output__is_valid is true.
|
||||
//
|
||||
// In any case, it always returns an output that satisfies both of:
|
||||
// - (output.code_point <= WUFFS_BASE__UNICODE_CODE_POINT__MAX_INCL).
|
||||
// - (output.byte_length <= s.len).
|
||||
//
|
||||
// If s is a sub-slice of a larger slice of valid UTF-8, but that sub-slice
|
||||
// boundary occurs in the middle of a multi-byte UTF-8 encoding of a single
|
||||
// code point, then this function may return something invalid. It is the
|
||||
// caller's responsibility to split on or otherwise manage UTF-8 boundaries.
|
||||
wuffs_base__utf_8__next__output //
|
||||
wuffs_base__utf_8__next(wuffs_base__slice_u8 s);
|
||||
|
||||
// wuffs_base__utf_8__longest_valid_prefix returns the largest n such that the
|
||||
// sub-slice s[..n] is valid UTF-8.
|
||||
//
|
||||
// In particular, it returns s.len if and only if all of s is valid UTF-8.
|
||||
//
|
||||
// If s is a sub-slice of a larger slice of valid UTF-8, but that sub-slice
|
||||
// boundary occurs in the middle of a multi-byte UTF-8 encoding of a single
|
||||
// code point, then this function will return less than s.len. It is the
|
||||
// caller's responsibility to split on or otherwise manage UTF-8 boundaries.
|
||||
size_t //
|
||||
wuffs_base__utf_8__longest_valid_prefix(wuffs_base__slice_u8 s);
|
||||
|
||||
// wuffs_base__ascii__longest_valid_prefix returns the largest n such that the
|
||||
// sub-slice s[..n] is valid ASCII.
|
||||
//
|
||||
// In particular, it returns s.len if and only if all of s is valid ASCII.
|
||||
// Equivalently, when none of the bytes in s have the 0x80 high bit set.
|
||||
size_t //
|
||||
wuffs_base__ascii__longest_valid_prefix(wuffs_base__slice_u8 s);
|
||||
|
||||
// ---------------- Interface Declarations.
|
||||
|
||||
extern const char* wuffs_base__hasher_u32__vtable_name;
|
||||
@@ -6633,6 +6756,8 @@ wuffs_base__io_writer__set(wuffs_base__io_buffer* b,
|
||||
|
||||
// ---------------- String Conversions
|
||||
|
||||
// ---------------- Unicode and UTF-8
|
||||
|
||||
// ----------------
|
||||
|
||||
#if !defined(WUFFS_CONFIG__MODULES) || defined(WUFFS_CONFIG__MODULE__BASE)
|
||||
@@ -8448,6 +8573,152 @@ fail_out_of_bounds:
|
||||
} while (0);
|
||||
}
|
||||
|
||||
// ---------------- Unicode and UTF-8
|
||||
|
||||
// wuffs_base__utf_8__byte_length_minus_1 is the byte length (minus 1) of a
|
||||
// UTF-8 encoded code point, based on the encoding's initial byte.
|
||||
// - 0x00 is 1-byte UTF-8 (ASCII).
|
||||
// - 0x01 is the start of 2-byte UTF-8.
|
||||
// - 0x02 is the start of 3-byte UTF-8.
|
||||
// - 0x03 is the start of 4-byte UTF-8.
|
||||
// - 0x40 is a UTF-8 tail byte.
|
||||
// - 0x80 is invalid UTF-8.
|
||||
//
|
||||
// RFC 3629 (UTF-8) gives this grammar for valid UTF-8:
|
||||
// UTF8-1 = %x00-7F
|
||||
// UTF8-2 = %xC2-DF UTF8-tail
|
||||
// UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) /
|
||||
// %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail )
|
||||
// UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) /
|
||||
// %xF4 %x80-8F 2( UTF8-tail )
|
||||
// UTF8-tail = %x80-BF
|
||||
static const uint8_t wuffs_base__utf_8__byte_length_minus_1[256] = {
|
||||
// 0 1 2 3 4 5 6 7
|
||||
// 8 9 A B C D E F
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x00 ..= 0x07.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x08 ..= 0x0F.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x10 ..= 0x17.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x18 ..= 0x1F.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20 ..= 0x27.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x28 ..= 0x2F.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x30 ..= 0x37.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x38 ..= 0x3F.
|
||||
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x40 ..= 0x47.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x48 ..= 0x4F.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x50 ..= 0x57.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x58 ..= 0x5F.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x60 ..= 0x67.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x68 ..= 0x6F.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x70 ..= 0x77.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x78 ..= 0x7F.
|
||||
|
||||
0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x80 ..= 0x87.
|
||||
0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x88 ..= 0x8F.
|
||||
0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x90 ..= 0x97.
|
||||
0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x98 ..= 0x9F.
|
||||
0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xA0 ..= 0xA7.
|
||||
0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xA8 ..= 0xAF.
|
||||
0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xB0 ..= 0xB7.
|
||||
0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xB8 ..= 0xBF.
|
||||
|
||||
0x80, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xC0 ..= 0xC7.
|
||||
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xC8 ..= 0xCF.
|
||||
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xD0 ..= 0xD7.
|
||||
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xD8 ..= 0xDF.
|
||||
0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, // 0xE0 ..= 0xE7.
|
||||
0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, // 0xE8 ..= 0xEF.
|
||||
0x03, 0x03, 0x03, 0x03, 0x03, 0x80, 0x80, 0x80, // 0xF0 ..= 0xF7.
|
||||
0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xF8 ..= 0xFF.
|
||||
// 0 1 2 3 4 5 6 7
|
||||
// 8 9 A B C D E F
|
||||
};
|
||||
|
||||
wuffs_base__utf_8__next__output //
|
||||
wuffs_base__utf_8__next(wuffs_base__slice_u8 s) {
|
||||
if (s.len == 0) {
|
||||
return wuffs_base__make_utf_8__next__output(0, 0);
|
||||
}
|
||||
uint32_t c = s.ptr[0];
|
||||
switch (wuffs_base__utf_8__byte_length_minus_1[c & 0xFF]) {
|
||||
case 0:
|
||||
return wuffs_base__make_utf_8__next__output(c, 1);
|
||||
|
||||
case 1:
|
||||
if (s.len < 2) {
|
||||
break;
|
||||
}
|
||||
c = wuffs_base__load_u16le__no_bounds_check(s.ptr);
|
||||
if ((c & 0xC000) != 0x8000) {
|
||||
break;
|
||||
}
|
||||
c = (0x0007C0 & (c << 6)) | (0x00003F & (c >> 8));
|
||||
return wuffs_base__make_utf_8__next__output(c, 2);
|
||||
|
||||
case 2:
|
||||
if (s.len < 3) {
|
||||
break;
|
||||
}
|
||||
c = wuffs_base__load_u24le__no_bounds_check(s.ptr);
|
||||
if ((c & 0xC0C000) != 0x808000) {
|
||||
break;
|
||||
}
|
||||
c = (0x00F000 & (c << 12)) | (0x000FC0 & (c >> 2)) |
|
||||
(0x00003F & (c >> 16));
|
||||
if ((c <= 0x07FF) || ((0xD800 <= c) && (c <= 0xDFFF))) {
|
||||
break;
|
||||
}
|
||||
return wuffs_base__make_utf_8__next__output(c, 3);
|
||||
|
||||
case 3:
|
||||
if (s.len < 4) {
|
||||
break;
|
||||
}
|
||||
c = wuffs_base__load_u32le__no_bounds_check(s.ptr);
|
||||
if ((c & 0xC0C0C000) != 0x80808000) {
|
||||
break;
|
||||
}
|
||||
c = (0x1C0000 & (c << 18)) | (0x03F000 & (c << 4)) |
|
||||
(0x000FC0 & (c >> 10)) | (0x00003F & (c >> 24));
|
||||
if ((c <= 0xFFFF) || (0x110000 <= c)) {
|
||||
break;
|
||||
}
|
||||
return wuffs_base__make_utf_8__next__output(c, 4);
|
||||
}
|
||||
|
||||
return wuffs_base__make_utf_8__next__output(
|
||||
WUFFS_BASE__UNICODE_REPLACEMENT_CHARACTER, 1);
|
||||
}
|
||||
|
||||
size_t //
|
||||
wuffs_base__utf_8__longest_valid_prefix(wuffs_base__slice_u8 s) {
|
||||
// TODO: possibly optimize the all-ASCII case (4 or 8 bytes at a time).
|
||||
//
|
||||
// TODO: possibly optimize this by manually inlining the
|
||||
// wuffs_base__utf_8__next calls.
|
||||
size_t original_len = s.len;
|
||||
while (s.len > 0) {
|
||||
wuffs_base__utf_8__next__output o = wuffs_base__utf_8__next(s);
|
||||
if ((o.code_point > 0x7F) && (o.byte_length == 1)) {
|
||||
break;
|
||||
}
|
||||
s.ptr += o.byte_length;
|
||||
s.len -= o.byte_length;
|
||||
}
|
||||
return original_len - s.len;
|
||||
}
|
||||
|
||||
size_t //
|
||||
wuffs_base__ascii__longest_valid_prefix(wuffs_base__slice_u8 s) {
|
||||
// TODO: possibly optimize this by checking 4 or 8 bytes at a time.
|
||||
uint8_t* original_ptr = s.ptr;
|
||||
uint8_t* p = s.ptr;
|
||||
uint8_t* q = s.ptr + s.len;
|
||||
for (; (p != q) && ((*p & 0x80) == 0); p++) {
|
||||
}
|
||||
return (size_t)(p - original_ptr);
|
||||
}
|
||||
|
||||
#endif // !defined(WUFFS_CONFIG__MODULES) ||
|
||||
// defined(WUFFS_CONFIG__MODULE__BASE)
|
||||
|
||||
|
||||
Reference in new issue
Block a user