diff --git a/example/jsonptr/jsonptr.cc b/example/jsonptr/jsonptr.cc index c7fe7802..b3790615 100644 --- a/example/jsonptr/jsonptr.cc +++ b/example/jsonptr/jsonptr.cc @@ -304,45 +304,21 @@ handle_unicode_code_point(uint32_t ucp) { } } - } else if (ucp <= 0x007F) { - switch (ucp) { - case '\"': - return write_dst("\\\"", 2); - case '\\': - return write_dst("\\\\", 2); - default: { - // The UTF-8 encoding takes 1 byte. - uint8_t esc0 = (uint8_t)(ucp); - return write_dst(&esc0, 1); - } + } else if (ucp == '\"') { + return write_dst("\\\"", 2); + + } else if (ucp == '\\') { + return write_dst("\\\\", 2); + + } else { + uint8_t u[WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL]; + size_t n = wuffs_base__utf_8__encode( + wuffs_base__make_slice_u8(&u[0], + WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL), + ucp); + if (n > 0) { + return write_dst(&u[0], n); } - - } else if (ucp <= 0x07FF) { - // The UTF-8 encoding takes 2 bytes. - uint8_t esc2[2]; - esc2[0] = 0xC0 | (uint8_t)((ucp >> 6)); - esc2[1] = 0x80 | (uint8_t)((ucp >> 0) & 0x3F); - return write_dst(&esc2[0], 2); - - } else if (ucp <= 0xFFFF) { - if ((0xD800 <= ucp) && (ucp <= 0xDFFF)) { - return "main: internal error: unexpected Unicode surrogate"; - } - // The UTF-8 encoding takes 3 bytes. - uint8_t esc3[3]; - esc3[0] = 0xE0 | (uint8_t)((ucp >> 12)); - esc3[1] = 0x80 | (uint8_t)((ucp >> 6) & 0x3F); - esc3[2] = 0x80 | (uint8_t)((ucp >> 0) & 0x3F); - return write_dst(&esc3[0], 3); - - } else if (ucp <= 0x10FFFF) { - // The UTF-8 encoding takes 4 bytes. - uint8_t esc4[4]; - esc4[0] = 0xF0 | (uint8_t)((ucp >> 18)); - esc4[1] = 0x80 | (uint8_t)((ucp >> 12) & 0x3F); - esc4[2] = 0x80 | (uint8_t)((ucp >> 6) & 0x3F); - esc4[3] = 0x80 | (uint8_t)((ucp >> 0) & 0x3F); - return write_dst(&esc4[0], 4); } return "main: internal error: unexpected Unicode code point"; diff --git a/internal/cgen/base/strconv-impl.c b/internal/cgen/base/strconv-impl.c index be5aa9af..1c2fbbe0 100644 --- a/internal/cgen/base/strconv-impl.c +++ b/internal/cgen/base/strconv-impl.c @@ -301,6 +301,42 @@ fail_out_of_bounds: // ---------------- Unicode and UTF-8 +size_t // +wuffs_base__utf_8__encode(wuffs_base__slice_u8 dst, uint32_t code_point) { + if (code_point <= 0x7F) { + if (dst.len >= 1) { + dst.ptr[0] = (uint8_t)(code_point); + return 1; + } + + } else if (code_point <= 0x07FF) { + if (dst.len >= 2) { + dst.ptr[0] = 0xC0 | (uint8_t)((code_point >> 6)); + dst.ptr[1] = 0x80 | (uint8_t)((code_point >> 0) & 0x3F); + return 2; + } + + } else if (code_point <= 0xFFFF) { + if ((dst.len >= 3) && ((code_point < 0xD800) || (0xDFFF < code_point))) { + dst.ptr[0] = 0xE0 | (uint8_t)((code_point >> 12)); + dst.ptr[1] = 0x80 | (uint8_t)((code_point >> 6) & 0x3F); + dst.ptr[2] = 0x80 | (uint8_t)((code_point >> 0) & 0x3F); + return 3; + } + + } else if (code_point <= 0x10FFFF) { + if (dst.len >= 4) { + dst.ptr[0] = 0xF0 | (uint8_t)((code_point >> 18)); + dst.ptr[1] = 0x80 | (uint8_t)((code_point >> 12) & 0x3F); + dst.ptr[2] = 0x80 | (uint8_t)((code_point >> 6) & 0x3F); + dst.ptr[3] = 0x80 | (uint8_t)((code_point >> 0) & 0x3F); + return 4; + } + } + + return 0; +} + // wuffs_base__utf_8__byte_length_minus_1 is the byte length (minus 1) of a // UTF-8 encoded code point, based on the encoding's initial byte. // - 0x00 is 1-byte UTF-8 (ASCII). diff --git a/internal/cgen/base/strconv-public.h b/internal/cgen/base/strconv-public.h index d4e394d4..b1f80780 100644 --- a/internal/cgen/base/strconv-public.h +++ b/internal/cgen/base/strconv-public.h @@ -133,6 +133,15 @@ wuffs_base__utf_8__next__output::is_valid() const { // -------- +// wuffs_base__utf_8__encode writes the UTF-8 encoding of code_point to s and +// returns the number of bytes written. If code_point is invalid, or if s is +// shorter than the entire encoding, it returns 0 (and no bytes are written). +// +// s will never be too short if its length is at least 4, also known as +// WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL. +size_t // +wuffs_base__utf_8__encode(wuffs_base__slice_u8 dst, uint32_t code_point); + // wuffs_base__utf_8__next returns the next UTF-8 code point (and that code // point's byte length) at the start of s. // diff --git a/internal/cgen/data.go b/internal/cgen/data.go index 5271f03a..ed2eb66a 100644 --- a/internal/cgen/data.go +++ b/internal/cgen/data.go @@ -79,12 +79,13 @@ const baseStrConvImplC = "" + "wuffs_base__parse_number__hexadecimal_digits[*p++];\n if (v == 0) {\n goto fail_bad_argument;\n }\n v &= 0x0F;\n\n for (; p < q; p++) {\n if (*p == '_') {\n continue;\n }\n uint8_t digit = wuffs_base__parse_number__hexadecimal_digits[*p];\n if (digit == 0) {\n goto fail_bad_argument;\n }\n digit &= 0x0F;\n if ((v >> 60) != 0) {\n goto fail_out_of_bounds;\n }\n v = (v << 4) | ((uint64_t)(digit));\n }\n\n wuffs_base__result_u64 ret;\n ret.status.repr = NULL;\n ret.value = v;\n return ret;\n } while (0);\n\nok_zero:\n do {\n wuffs_base__result_u64 ret;\n ret.status.repr = NULL;\n ret.value = 0;\n return ret;\n } while (0);\n\nfail_bad_argument:\n do {\n wuffs_base__result_u64 ret;\n ret.status.repr = wuffs_base__error__bad_argument;\n ret.value = 0;\n return ret;\n } while (0);\n\nfail_out_of_bounds:\n do {\n wuffs_base__result_u64 ret;\n ret.status.repr = wuffs_base__error__out_of_bounds;\n ret.value = 0;\n return ret;\n " + " } while (0);\n}\n\n" + "" + - "// ---------------- Unicode and UTF-8\n\n// wuffs_base__utf_8__byte_length_minus_1 is the byte length (minus 1) of a\n// UTF-8 encoded code point, based on the encoding's initial byte.\n// - 0x00 is 1-byte UTF-8 (ASCII).\n// - 0x01 is the start of 2-byte UTF-8.\n// - 0x02 is the start of 3-byte UTF-8.\n// - 0x03 is the start of 4-byte UTF-8.\n// - 0x40 is a UTF-8 tail byte.\n// - 0x80 is invalid UTF-8.\n//\n// RFC 3629 (UTF-8) gives this grammar for valid UTF-8:\n// UTF8-1 = %x00-7F\n// UTF8-2 = %xC2-DF UTF8-tail\n// UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) /\n// %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail )\n// UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) /\n// %xF4 %x80-8F 2( UTF8-tail )\n// UTF8-tail = %x80-BF\nstatic const uint8_t wuffs_base__utf_8__byte_length_minus_1[256] = {\n // 0 1 2 3 4 5 6 7\n // 8 9 A B C D E F\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x" + - "00, 0x00, 0x00, // 0x00 ..= 0x07.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x08 ..= 0x0F.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x10 ..= 0x17.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x18 ..= 0x1F.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20 ..= 0x27.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x28 ..= 0x2F.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x30 ..= 0x37.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x38 ..= 0x3F.\n\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x40 ..= 0x47.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x48 ..= 0x4F.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x50 ..= 0x57.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x58 ..= 0x5F.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x60 ..= 0x67.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x68 ..= 0x6F.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x70 ..= " + - "0x77.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x78 ..= 0x7F.\n\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x80 ..= 0x87.\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x88 ..= 0x8F.\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x90 ..= 0x97.\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x98 ..= 0x9F.\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xA0 ..= 0xA7.\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xA8 ..= 0xAF.\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xB0 ..= 0xB7.\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xB8 ..= 0xBF.\n\n 0x80, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xC0 ..= 0xC7.\n 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xC8 ..= 0xCF.\n 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xD0 ..= 0xD7.\n 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xD8 ..= 0xDF.\n 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, // 0xE0 ..= 0xE7.\n 0x02, 0x02, 0x02, " + - "0x02, 0x02, 0x02, 0x02, 0x02, // 0xE8 ..= 0xEF.\n 0x03, 0x03, 0x03, 0x03, 0x03, 0x80, 0x80, 0x80, // 0xF0 ..= 0xF7.\n 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xF8 ..= 0xFF.\n // 0 1 2 3 4 5 6 7\n // 8 9 A B C D E F\n};\n\nwuffs_base__utf_8__next__output //\nwuffs_base__utf_8__next(wuffs_base__slice_u8 s) {\n if (s.len == 0) {\n return wuffs_base__make_utf_8__next__output(0, 0);\n }\n uint32_t c = s.ptr[0];\n switch (wuffs_base__utf_8__byte_length_minus_1[c & 0xFF]) {\n case 0:\n return wuffs_base__make_utf_8__next__output(c, 1);\n\n case 1:\n if (s.len < 2) {\n break;\n }\n c = wuffs_base__load_u16le__no_bounds_check(s.ptr);\n if ((c & 0xC000) != 0x8000) {\n break;\n }\n c = (0x0007C0 & (c << 6)) | (0x00003F & (c >> 8));\n return wuffs_base__make_utf_8__next__output(c, 2);\n\n case 2:\n if (s.len < 3) {\n break;\n }\n c = wuffs_base__load_u24le__no_bounds_check(s.ptr);" + - "\n if ((c & 0xC0C000) != 0x808000) {\n break;\n }\n c = (0x00F000 & (c << 12)) | (0x000FC0 & (c >> 2)) |\n (0x00003F & (c >> 16));\n if ((c <= 0x07FF) || ((0xD800 <= c) && (c <= 0xDFFF))) {\n break;\n }\n return wuffs_base__make_utf_8__next__output(c, 3);\n\n case 3:\n if (s.len < 4) {\n break;\n }\n c = wuffs_base__load_u32le__no_bounds_check(s.ptr);\n if ((c & 0xC0C0C000) != 0x80808000) {\n break;\n }\n c = (0x1C0000 & (c << 18)) | (0x03F000 & (c << 4)) |\n (0x000FC0 & (c >> 10)) | (0x00003F & (c >> 24));\n if ((c <= 0xFFFF) || (0x110000 <= c)) {\n break;\n }\n return wuffs_base__make_utf_8__next__output(c, 4);\n }\n\n return wuffs_base__make_utf_8__next__output(\n WUFFS_BASE__UNICODE_REPLACEMENT_CHARACTER, 1);\n}\n\nsize_t //\nwuffs_base__utf_8__longest_valid_prefix(wuffs_base__slice_u8 s) {\n // TODO: possibly optimize the all-ASCII case (4 or 8 bytes at a time).\n //\n // TODO: possibly optimize thi" + - "s by manually inlining the\n // wuffs_base__utf_8__next calls.\n size_t original_len = s.len;\n while (s.len > 0) {\n wuffs_base__utf_8__next__output o = wuffs_base__utf_8__next(s);\n if ((o.code_point > 0x7F) && (o.byte_length == 1)) {\n break;\n }\n s.ptr += o.byte_length;\n s.len -= o.byte_length;\n }\n return original_len - s.len;\n}\n\nsize_t //\nwuffs_base__ascii__longest_valid_prefix(wuffs_base__slice_u8 s) {\n // TODO: possibly optimize this by checking 4 or 8 bytes at a time.\n uint8_t* original_ptr = s.ptr;\n uint8_t* p = s.ptr;\n uint8_t* q = s.ptr + s.len;\n for (; (p != q) && ((*p & 0x80) == 0); p++) {\n }\n return (size_t)(p - original_ptr);\n}\n" + + "// ---------------- Unicode and UTF-8\n\nsize_t //\nwuffs_base__utf_8__encode(wuffs_base__slice_u8 dst, uint32_t code_point) {\n if (code_point <= 0x7F) {\n if (dst.len >= 1) {\n dst.ptr[0] = (uint8_t)(code_point);\n return 1;\n }\n\n } else if (code_point <= 0x07FF) {\n if (dst.len >= 2) {\n dst.ptr[0] = 0xC0 | (uint8_t)((code_point >> 6));\n dst.ptr[1] = 0x80 | (uint8_t)((code_point >> 0) & 0x3F);\n return 2;\n }\n\n } else if (code_point <= 0xFFFF) {\n if ((dst.len >= 3) && ((code_point < 0xD800) || (0xDFFF < code_point))) {\n dst.ptr[0] = 0xE0 | (uint8_t)((code_point >> 12));\n dst.ptr[1] = 0x80 | (uint8_t)((code_point >> 6) & 0x3F);\n dst.ptr[2] = 0x80 | (uint8_t)((code_point >> 0) & 0x3F);\n return 3;\n }\n\n } else if (code_point <= 0x10FFFF) {\n if (dst.len >= 4) {\n dst.ptr[0] = 0xF0 | (uint8_t)((code_point >> 18));\n dst.ptr[1] = 0x80 | (uint8_t)((code_point >> 12) & 0x3F);\n dst.ptr[2] = 0x80 | (uint8_t)((code_point >> 6) & 0x3F);\n dst.p" + + "tr[3] = 0x80 | (uint8_t)((code_point >> 0) & 0x3F);\n return 4;\n }\n }\n\n return 0;\n}\n\n// wuffs_base__utf_8__byte_length_minus_1 is the byte length (minus 1) of a\n// UTF-8 encoded code point, based on the encoding's initial byte.\n// - 0x00 is 1-byte UTF-8 (ASCII).\n// - 0x01 is the start of 2-byte UTF-8.\n// - 0x02 is the start of 3-byte UTF-8.\n// - 0x03 is the start of 4-byte UTF-8.\n// - 0x40 is a UTF-8 tail byte.\n// - 0x80 is invalid UTF-8.\n//\n// RFC 3629 (UTF-8) gives this grammar for valid UTF-8:\n// UTF8-1 = %x00-7F\n// UTF8-2 = %xC2-DF UTF8-tail\n// UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) /\n// %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail )\n// UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) /\n// %xF4 %x80-8F 2( UTF8-tail )\n// UTF8-tail = %x80-BF\nstatic const uint8_t wuffs_base__utf_8__byte_length_minus_1[256] = {\n // 0 1 2 3 4 5 6 7\n // 8 9 A B C" + + " D E F\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x00 ..= 0x07.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x08 ..= 0x0F.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x10 ..= 0x17.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x18 ..= 0x1F.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x20 ..= 0x27.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x28 ..= 0x2F.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x30 ..= 0x37.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x38 ..= 0x3F.\n\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x40 ..= 0x47.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x48 ..= 0x4F.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x50 ..= 0x57.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x58 ..= 0x5F.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x60 ..= 0x67.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x68 ..= 0x6F.\n 0x00, " + + "0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x70 ..= 0x77.\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x78 ..= 0x7F.\n\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x80 ..= 0x87.\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x88 ..= 0x8F.\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x90 ..= 0x97.\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x98 ..= 0x9F.\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xA0 ..= 0xA7.\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xA8 ..= 0xAF.\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xB0 ..= 0xB7.\n 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0xB8 ..= 0xBF.\n\n 0x80, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xC0 ..= 0xC7.\n 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xC8 ..= 0xCF.\n 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xD0 ..= 0xD7.\n 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // 0xD8 ..= 0xDF.\n 0x02, 0x02, 0x02, 0x02, 0x02, 0x02" + + ", 0x02, 0x02, // 0xE0 ..= 0xE7.\n 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, // 0xE8 ..= 0xEF.\n 0x03, 0x03, 0x03, 0x03, 0x03, 0x80, 0x80, 0x80, // 0xF0 ..= 0xF7.\n 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // 0xF8 ..= 0xFF.\n // 0 1 2 3 4 5 6 7\n // 8 9 A B C D E F\n};\n\nwuffs_base__utf_8__next__output //\nwuffs_base__utf_8__next(wuffs_base__slice_u8 s) {\n if (s.len == 0) {\n return wuffs_base__make_utf_8__next__output(0, 0);\n }\n uint32_t c = s.ptr[0];\n switch (wuffs_base__utf_8__byte_length_minus_1[c & 0xFF]) {\n case 0:\n return wuffs_base__make_utf_8__next__output(c, 1);\n\n case 1:\n if (s.len < 2) {\n break;\n }\n c = wuffs_base__load_u16le__no_bounds_check(s.ptr);\n if ((c & 0xC000) != 0x8000) {\n break;\n }\n c = (0x0007C0 & (c << 6)) | (0x00003F & (c >> 8));\n return wuffs_base__make_utf_8__next__output(c, 2);\n\n case 2:\n if (s.len < 3) {\n break;\n }\n " + + " c = wuffs_base__load_u24le__no_bounds_check(s.ptr);\n if ((c & 0xC0C000) != 0x808000) {\n break;\n }\n c = (0x00F000 & (c << 12)) | (0x000FC0 & (c >> 2)) |\n (0x00003F & (c >> 16));\n if ((c <= 0x07FF) || ((0xD800 <= c) && (c <= 0xDFFF))) {\n break;\n }\n return wuffs_base__make_utf_8__next__output(c, 3);\n\n case 3:\n if (s.len < 4) {\n break;\n }\n c = wuffs_base__load_u32le__no_bounds_check(s.ptr);\n if ((c & 0xC0C0C000) != 0x80808000) {\n break;\n }\n c = (0x1C0000 & (c << 18)) | (0x03F000 & (c << 4)) |\n (0x000FC0 & (c >> 10)) | (0x00003F & (c >> 24));\n if ((c <= 0xFFFF) || (0x110000 <= c)) {\n break;\n }\n return wuffs_base__make_utf_8__next__output(c, 4);\n }\n\n return wuffs_base__make_utf_8__next__output(\n WUFFS_BASE__UNICODE_REPLACEMENT_CHARACTER, 1);\n}\n\nsize_t //\nwuffs_base__utf_8__longest_valid_prefix(wuffs_base__slice_u8 s) {\n // TODO: possibly optimize the all-ASCII case (4 or 8 " + + "bytes at a time).\n //\n // TODO: possibly optimize this by manually inlining the\n // wuffs_base__utf_8__next calls.\n size_t original_len = s.len;\n while (s.len > 0) {\n wuffs_base__utf_8__next__output o = wuffs_base__utf_8__next(s);\n if ((o.code_point > 0x7F) && (o.byte_length == 1)) {\n break;\n }\n s.ptr += o.byte_length;\n s.len -= o.byte_length;\n }\n return original_len - s.len;\n}\n\nsize_t //\nwuffs_base__ascii__longest_valid_prefix(wuffs_base__slice_u8 s) {\n // TODO: possibly optimize this by checking 4 or 8 bytes at a time.\n uint8_t* original_ptr = s.ptr;\n uint8_t* p = s.ptr;\n uint8_t* q = s.ptr + s.len;\n for (; (p != q) && ((*p & 0x80) == 0); p++) {\n }\n return (size_t)(p - original_ptr);\n}\n" + "" const baseCorePrivateH = "" + @@ -296,9 +297,9 @@ const baseStrConvPublicH = "" + "// --------\n\n// wuffs_base__utf_8__next__output is the type returned by\n// wuffs_base__utf_8__next.\ntypedef struct {\n uint32_t code_point;\n uint32_t byte_length;\n\n#ifdef __cplusplus\n inline bool is_valid() const;\n#endif // __cplusplus\n\n} wuffs_base__utf_8__next__output;\n\nstatic inline wuffs_base__utf_8__next__output //\nwuffs_base__make_utf_8__next__output(uint32_t code_point,\n uint32_t byte_length) {\n wuffs_base__utf_8__next__output ret;\n ret.code_point = code_point;\n ret.byte_length = byte_length;\n return ret;\n}\n\nstatic inline bool //\nwuffs_base__utf_8__next__output__is_valid(\n const wuffs_base__utf_8__next__output* o) {\n if (o) {\n uint32_t cp = o->code_point;\n switch (o->byte_length) {\n case 1:\n return (cp <= 0x7F);\n case 2:\n return (0x080 <= cp) && (cp <= 0x7FF);\n case 3:\n // Avoid the 0xD800 ..= 0xDFFF surrogate range.\n return ((0x0800 <= cp) && (cp <= 0xD7FF)) ||\n ((0xE000 <= cp) && (cp <= 0xF" + "FFF));\n case 4:\n return (0x00010000 <= cp) && (cp <= 0x0010FFFF);\n }\n }\n return false;\n}\n\n#ifdef __cplusplus\n\ninline bool //\nwuffs_base__utf_8__next__output::is_valid() const {\n return wuffs_base__utf_8__next__output__is_valid(this);\n}\n\n#endif // __cplusplus\n\n" + "" + - "// --------\n\n// wuffs_base__utf_8__next returns the next UTF-8 code point (and that code\n// point's byte length) at the start of s.\n//\n// There are exactly two cases in which this function returns something where\n// wuffs_base__utf_8__next__output__is_valid is false:\n// - If s is empty then it returns {.code_point=0, .byte_length=0}.\n// - If s is non-empty and starts with invalid UTF-8 then it returns\n// {.code_point=WUFFS_BASE__UNICODE_REPLACEMENT_CHARACTER, .byte_length=1}.\n//\n// Otherwise, it returns something where\n// wuffs_base__utf_8__next__output__is_valid is true.\n//\n// In any case, it always returns an output that satisfies both of:\n// - (output.code_point <= WUFFS_BASE__UNICODE_CODE_POINT__MAX_INCL).\n// - (output.byte_length <= s.len).\n//\n// If s is a sub-slice of a larger slice of valid UTF-8, but that sub-slice\n// boundary occurs in the middle of a multi-byte UTF-8 encoding of a single\n// code point, then this function may return something invalid. It is the\n// caller's responsibility to s" + - "plit on or otherwise manage UTF-8 boundaries.\nwuffs_base__utf_8__next__output //\nwuffs_base__utf_8__next(wuffs_base__slice_u8 s);\n\n// wuffs_base__utf_8__longest_valid_prefix returns the largest n such that the\n// sub-slice s[..n] is valid UTF-8.\n//\n// In particular, it returns s.len if and only if all of s is valid UTF-8.\n//\n// If s is a sub-slice of a larger slice of valid UTF-8, but that sub-slice\n// boundary occurs in the middle of a multi-byte UTF-8 encoding of a single\n// code point, then this function will return less than s.len. It is the\n// caller's responsibility to split on or otherwise manage UTF-8 boundaries.\nsize_t //\nwuffs_base__utf_8__longest_valid_prefix(wuffs_base__slice_u8 s);\n\n// wuffs_base__ascii__longest_valid_prefix returns the largest n such that the\n// sub-slice s[..n] is valid ASCII.\n//\n// In particular, it returns s.len if and only if all of s is valid ASCII.\n// Equivalently, when none of the bytes in s have the 0x80 high bit set.\nsize_t //\nwuffs_base__ascii__longest_valid_prefix(" + - "wuffs_base__slice_u8 s);\n" + + "// --------\n\n// wuffs_base__utf_8__encode writes the UTF-8 encoding of code_point to s and\n// returns the number of bytes written. If code_point is invalid, or if s is\n// shorter than the entire encoding, it returns 0 (and no bytes are written).\n//\n// s will never be too short if its length is at least 4, also known as\n// WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL.\nsize_t //\nwuffs_base__utf_8__encode(wuffs_base__slice_u8 dst, uint32_t code_point);\n\n// wuffs_base__utf_8__next returns the next UTF-8 code point (and that code\n// point's byte length) at the start of s.\n//\n// There are exactly two cases in which this function returns something where\n// wuffs_base__utf_8__next__output__is_valid is false:\n// - If s is empty then it returns {.code_point=0, .byte_length=0}.\n// - If s is non-empty and starts with invalid UTF-8 then it returns\n// {.code_point=WUFFS_BASE__UNICODE_REPLACEMENT_CHARACTER, .byte_length=1}.\n//\n// Otherwise, it returns something where\n// wuffs_base__utf_8__next__output__is_valid is true.\n//" + + "\n// In any case, it always returns an output that satisfies both of:\n// - (output.code_point <= WUFFS_BASE__UNICODE_CODE_POINT__MAX_INCL).\n// - (output.byte_length <= s.len).\n//\n// If s is a sub-slice of a larger slice of valid UTF-8, but that sub-slice\n// boundary occurs in the middle of a multi-byte UTF-8 encoding of a single\n// code point, then this function may return something invalid. It is the\n// caller's responsibility to split on or otherwise manage UTF-8 boundaries.\nwuffs_base__utf_8__next__output //\nwuffs_base__utf_8__next(wuffs_base__slice_u8 s);\n\n// wuffs_base__utf_8__longest_valid_prefix returns the largest n such that the\n// sub-slice s[..n] is valid UTF-8.\n//\n// In particular, it returns s.len if and only if all of s is valid UTF-8.\n//\n// If s is a sub-slice of a larger slice of valid UTF-8, but that sub-slice\n// boundary occurs in the middle of a multi-byte UTF-8 encoding of a single\n// code point, then this function will return less than s.len. It is the\n// caller's responsibility to spl" + + "it on or otherwise manage UTF-8 boundaries.\nsize_t //\nwuffs_base__utf_8__longest_valid_prefix(wuffs_base__slice_u8 s);\n\n// wuffs_base__ascii__longest_valid_prefix returns the largest n such that the\n// sub-slice s[..n] is valid ASCII.\n//\n// In particular, it returns s.len if and only if all of s is valid ASCII.\n// Equivalently, when none of the bytes in s have the 0x80 high bit set.\nsize_t //\nwuffs_base__ascii__longest_valid_prefix(wuffs_base__slice_u8 s);\n" + "" const baseTokenPrivateH = "" + diff --git a/release/c/wuffs-unsupported-snapshot.c b/release/c/wuffs-unsupported-snapshot.c index 2c66a3b9..597b1098 100644 --- a/release/c/wuffs-unsupported-snapshot.c +++ b/release/c/wuffs-unsupported-snapshot.c @@ -3512,6 +3512,15 @@ wuffs_base__utf_8__next__output::is_valid() const { // -------- +// wuffs_base__utf_8__encode writes the UTF-8 encoding of code_point to s and +// returns the number of bytes written. If code_point is invalid, or if s is +// shorter than the entire encoding, it returns 0 (and no bytes are written). +// +// s will never be too short if its length is at least 4, also known as +// WUFFS_BASE__UTF_8__BYTE_LENGTH__MAX_INCL. +size_t // +wuffs_base__utf_8__encode(wuffs_base__slice_u8 dst, uint32_t code_point); + // wuffs_base__utf_8__next returns the next UTF-8 code point (and that code // point's byte length) at the start of s. // @@ -8612,6 +8621,42 @@ fail_out_of_bounds: // ---------------- Unicode and UTF-8 +size_t // +wuffs_base__utf_8__encode(wuffs_base__slice_u8 dst, uint32_t code_point) { + if (code_point <= 0x7F) { + if (dst.len >= 1) { + dst.ptr[0] = (uint8_t)(code_point); + return 1; + } + + } else if (code_point <= 0x07FF) { + if (dst.len >= 2) { + dst.ptr[0] = 0xC0 | (uint8_t)((code_point >> 6)); + dst.ptr[1] = 0x80 | (uint8_t)((code_point >> 0) & 0x3F); + return 2; + } + + } else if (code_point <= 0xFFFF) { + if ((dst.len >= 3) && ((code_point < 0xD800) || (0xDFFF < code_point))) { + dst.ptr[0] = 0xE0 | (uint8_t)((code_point >> 12)); + dst.ptr[1] = 0x80 | (uint8_t)((code_point >> 6) & 0x3F); + dst.ptr[2] = 0x80 | (uint8_t)((code_point >> 0) & 0x3F); + return 3; + } + + } else if (code_point <= 0x10FFFF) { + if (dst.len >= 4) { + dst.ptr[0] = 0xF0 | (uint8_t)((code_point >> 18)); + dst.ptr[1] = 0x80 | (uint8_t)((code_point >> 12) & 0x3F); + dst.ptr[2] = 0x80 | (uint8_t)((code_point >> 6) & 0x3F); + dst.ptr[3] = 0x80 | (uint8_t)((code_point >> 0) & 0x3F); + return 4; + } + } + + return 0; +} + // wuffs_base__utf_8__byte_length_minus_1 is the byte length (minus 1) of a // UTF-8 encoded code point, based on the encoding's initial byte. // - 0x00 is 1-byte UTF-8 (ASCII).