feat(unicode,datastruct): port Unicode tables and data structures

ghostty-unicode: Ghostty computes codepoint properties with the uucode
Zig package (v0.2.0, Unicode 17.0.0). There is no Rust equivalent, so
a trimmed UCD set is vendored in ucd/ and build.rs re-implements
uucode's derivations: cell width rules incl. Ghostty's overrides for
emoji modifiers and Prepend, grapheme break classes with the
compile-time grapheme_break state table, the is_symbol set, and emoji
presentation. Lookups are three array indexes per codepoint. Validated
against the official GraphemeBreakTest.txt.

ghostty-datastruct: split_tree (used for GTK splits; f16 ratios so
rounding and debug diagrams match Zig), circ_buf, lru, cache_table,
blocking_queue, segmented_pool, message_data, comparison,
array_list_collection. intrusive_linked_list is not needed (lru is
index based). lru now reports the evicted (old) key, Zig reported the
new key, which appears to be a bug.

Not yet covered: case_folding_full (needed by input bindings).

Test Plan:
- cargo test -p ghostty-unicode (16) -p ghostty-datastruct (88)
- cargo clippy for both crates with -D warnings

crates/ghostty-unicode/src/emoji_table.rs

@@ -0,0 +1,19 @@
+//! The generated `is_emoji_presentation` lookup table (uucode runtime field, used by the font codepoint resolver; no Zig counterpart file). Generated by
+//! `build.rs`.
+
+use crate::lut::Tables;
+
+mod generated {
+    include!(concat!(env!("OUT_DIR"), "/emoji_tables.rs"));
+}
+
+/// The lookup table (codepoint -> bool).
+#[inline]
+#[must_use]
+pub fn table() -> Tables<'static, bool> {
+    Tables {
+        stage1: &generated::STAGE1,
+        stage2: &generated::STAGE2,
+        stage3: &generated::STAGE3,
+    }
+}

crates/ghostty-unicode/src/grapheme.rs

@@ -0,0 +1,499 @@
+//! Grapheme break detection (`grapheme.zig`), backed by uucode's
+//! `x.grapheme.computeGraphemeBreakNoControl` precomputed for every
+//! combination of state and break properties.
+
+use crate::props::GraphemeBreakNoControl as Gb;
+use crate::props_table::table;
+
+/// State carried between sequential [`grapheme_break`] calls
+/// (`uucode.grapheme.BreakState`).
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
+#[repr(u8)]
+pub enum BreakState {
+    #[default]
+    Default = 0,
+    RegionalIndicator,
+    ExtendedPictographic,
+    IndicConjunctBreakConsonant,
+    IndicConjunctBreakLinker,
+}
+
+const ALL_STATES: [BreakState; 5] = [
+    BreakState::Default,
+    BreakState::RegionalIndicator,
+    BreakState::ExtendedPictographic,
+    BreakState::IndicConjunctBreakConsonant,
+    BreakState::IndicConjunctBreakLinker,
+];
+
+const ALL_GB: [Gb; Gb::COUNT] = [
+    Gb::Other,
+    Gb::Prepend,
+    Gb::RegionalIndicator,
+    Gb::SpacingMark,
+    Gb::L,
+    Gb::V,
+    Gb::T,
+    Gb::Lv,
+    Gb::Lvt,
+    Gb::Zwj,
+    Gb::Zwnj,
+    Gb::ExtendedPictographic,
+    Gb::EmojiModifierBase,
+    Gb::EmojiModifier,
+    Gb::IndicConjunctBreakExtend,
+    Gb::IndicConjunctBreakLinker,
+    Gb::IndicConjunctBreakConsonant,
+];
+
+const fn is_indic_conjunct_break_extend(gb: Gb) -> bool {
+    matches!(gb, Gb::IndicConjunctBreakExtend | Gb::Zwj)
+}
+
+// Despite `emoji_modifier` being `extend` according to
+// GraphemeBreakProperty.txt and UAX #29 (in addition to tests in
+// GraphemeBreakTest.txt), UTS #51 states: `emoji_modifier_sequence :=
+// emoji_modifier_base emoji_modifier` in ED-13 (emoji modifier sequence) under
+// 1.4.4 (Emoji Modifiers), and: "When used alone, the default representation
+// of these modifier characters is a color swatch... To have an effect on an
+// emoji, an emoji modifier must immediately follow that base emoji
+// character." in 2.4 (Diversity). Additionally it states "Skin tone
+// modifiers and hair components should be displayed even in isolation" in
+// ED-20 (basic emoji set) under 1.4.6 (Emoji Sets). Here we decide to diverge
+// from the grapheme break spec, which is allowed under "tailored" grapheme
+// clusters.
+const fn is_extend(gb: Gb) -> bool {
+    matches!(
+        gb,
+        Gb::Zwnj | Gb::IndicConjunctBreakExtend | Gb::IndicConjunctBreakLinker
+    )
+}
+
+const fn is_extended_pictographic(gb: Gb) -> bool {
+    matches!(gb, Gb::ExtendedPictographic | Gb::EmojiModifierBase)
+}
+
+/// Port of uucode `x.grapheme.computeGraphemeBreakNoControl`: the rules for
+/// `control`, `cr`, and `lf` are ignored, since they are assumed to have been
+/// handled prior or stripped from the input.
+#[must_use]
+pub const fn compute_grapheme_break_no_control(gb1: Gb, gb2: Gb, state: &mut BreakState) -> bool {
+    // Set state back to default when `gb1` or `gb2` is not expected in sequence.
+    match *state {
+        BreakState::RegionalIndicator => {
+            if !matches!(gb1, Gb::RegionalIndicator) || !matches!(gb2, Gb::RegionalIndicator) {
+                *state = BreakState::Default;
+            }
+        }
+        BreakState::ExtendedPictographic => {
+            if !matches!(
+                gb1,
+                Gb::IndicConjunctBreakExtend
+                    | Gb::IndicConjunctBreakLinker
+                    | Gb::Zwnj
+                    | Gb::Zwj
+                    | Gb::ExtendedPictographic
+                    | Gb::EmojiModifierBase
+                    | Gb::EmojiModifier
+            ) {
+                *state = BreakState::Default;
+            }
+            if !matches!(
+                gb2,
+                Gb::IndicConjunctBreakExtend
+                    | Gb::IndicConjunctBreakLinker
+                    | Gb::Zwnj
+                    | Gb::Zwj
+                    | Gb::ExtendedPictographic
+                    | Gb::EmojiModifierBase
+                    | Gb::EmojiModifier
+            ) {
+                *state = BreakState::Default;
+            }
+        }
+        BreakState::IndicConjunctBreakConsonant | BreakState::IndicConjunctBreakLinker => {
+            if !matches!(
+                gb1,
+                Gb::IndicConjunctBreakConsonant
+                    | Gb::IndicConjunctBreakLinker
+                    | Gb::IndicConjunctBreakExtend
+                    | Gb::Zwj
+            ) {
+                *state = BreakState::Default;
+            }
+            if !matches!(
+                gb2,
+                Gb::IndicConjunctBreakConsonant
+                    | Gb::IndicConjunctBreakLinker
+                    | Gb::IndicConjunctBreakExtend
+                    | Gb::Zwj
+            ) {
+                *state = BreakState::Default;
+            }
+        }
+        BreakState::Default => {}
+    }
+
+    // GB6: L x (L | V | LV | VT)
+    if matches!(gb1, Gb::L) && matches!(gb2, Gb::L | Gb::V | Gb::Lv | Gb::Lvt) {
+        return false;
+    }
+
+    // GB7: (LV | V) x (V | T)
+    if matches!(gb1, Gb::Lv | Gb::V) && matches!(gb2, Gb::V | Gb::T) {
+        return false;
+    }
+
+    // GB8: (LVT | T) x T
+    if matches!(gb1, Gb::Lvt | Gb::T) && matches!(gb2, Gb::T) {
+        return false;
+    }
+
+    // Handle GB9 (Extend | ZWJ) later, since it can also match the start of
+    // GB9c (Indic) and GB11 (Emoji ZWJ)
+
+    // GB9a: SpacingMark
+    if matches!(gb2, Gb::SpacingMark) {
+        return false;
+    }
+
+    // GB9b: Prepend
+    if matches!(gb1, Gb::Prepend) {
+        return false;
+    }
+
+    // GB9c: Indic
+    if matches!(gb1, Gb::IndicConjunctBreakConsonant) {
+        // start of sequence:
+        if is_indic_conjunct_break_extend(gb2) {
+            *state = BreakState::IndicConjunctBreakConsonant;
+            return false;
+        } else if matches!(gb2, Gb::IndicConjunctBreakLinker) {
+            // jump straight to linker state
+            *state = BreakState::IndicConjunctBreakLinker;
+            return false;
+        }
+        // else, not an Indic sequence
+    } else if matches!(*state, BreakState::IndicConjunctBreakConsonant) {
+        // consonant state:
+        if matches!(gb2, Gb::IndicConjunctBreakLinker) {
+            // consonant -> linker transition
+            *state = BreakState::IndicConjunctBreakLinker;
+            return false;
+        } else if is_indic_conjunct_break_extend(gb2) {
+            // continue [extend]* sequence
+            return false;
+        }
+        // Not a valid Indic sequence
+        *state = BreakState::Default;
+    } else if matches!(*state, BreakState::IndicConjunctBreakLinker) {
+        // linker state:
+        if matches!(gb2, Gb::IndicConjunctBreakLinker) || is_indic_conjunct_break_extend(gb2) {
+            // continue [extend linker]* sequence
+            return false;
+        } else if matches!(gb2, Gb::IndicConjunctBreakConsonant) {
+            // linker -> end of sequence
+            *state = BreakState::Default;
+            return false;
+        }
+        // Not a valid Indic sequence
+        *state = BreakState::Default;
+    }
+
+    // GB11: Emoji ZWJ sequence and Emoji modifier sequence
+    if is_extended_pictographic(gb1) {
+        // start of sequence:
+        if is_extend(gb2) || matches!(gb2, Gb::Zwj) {
+            *state = BreakState::ExtendedPictographic;
+            return false;
+        }
+
+        // The `emoji_modifier_sequence` case is described in the comment for
+        // `is_extend` above, from UTS #51.
+        if matches!(gb1, Gb::EmojiModifierBase) && matches!(gb2, Gb::EmojiModifier) {
+            *state = BreakState::ExtendedPictographic;
+            return false;
+        }
+        // else, not an Emoji ZWJ sequence
+    } else if matches!(*state, BreakState::ExtendedPictographic) {
+        // continue or end sequence:
+        if (is_extend(gb1) || matches!(gb1, Gb::EmojiModifier))
+            && (is_extend(gb2) || matches!(gb2, Gb::Zwj))
+        {
+            // continue extend* ZWJ sequence
+            return false;
+        } else if matches!(gb1, Gb::Zwj) && is_extended_pictographic(gb2) {
+            // ZWJ -> end of sequence
+            *state = BreakState::Default;
+            return false;
+        }
+        // Not a valid Emoji ZWJ sequence
+        *state = BreakState::Default;
+    }
+
+    // GB12 and GB13: Regional Indicator
+    if matches!(gb1, Gb::RegionalIndicator) && matches!(gb2, Gb::RegionalIndicator) {
+        if matches!(*state, BreakState::Default) {
+            *state = BreakState::RegionalIndicator;
+            return false;
+        }
+        *state = BreakState::Default;
+        return true;
+    }
+
+    // GB9: x (Extend | ZWJ)
+    if is_extend(gb2) || matches!(gb2, Gb::Zwj) {
+        return false;
+    }
+
+    // GB999: Otherwise, break everywhere
+    true
+}
+
+/// Result of one table lookup: the break decision and the next state.
+#[derive(Debug, Clone, Copy)]
+struct Value {
+    result: bool,
+    state: BreakState,
+}
+
+const N_GB: usize = Gb::COUNT;
+const N_STATE: usize = 5;
+
+/// Precomputed lookup table for all possible permutations of state and
+/// grapheme break properties (5 states x 17 x 17 = 1445 entries).
+static PRECOMPUTE: [Value; N_STATE * N_GB * N_GB] = {
+    let mut data = [Value {
+        result: false,
+        state: BreakState::Default,
+    }; N_STATE * N_GB * N_GB];
+    let mut s = 0;
+    while s < N_STATE {
+        let mut a = 0;
+        while a < N_GB {
+            let mut b = 0;
+            while b < N_GB {
+                let mut state = ALL_STATES[s];
+                let result = compute_grapheme_break_no_control(ALL_GB[a], ALL_GB[b], &mut state);
+                data[index(ALL_GB[a], ALL_GB[b], ALL_STATES[s])] = Value { result, state };
+                b += 1;
+            }
+            a += 1;
+        }
+        s += 1;
+    }
+    data
+};
+
+const fn index(gb1: Gb, gb2: Gb, state: BreakState) -> usize {
+    (state as usize) * N_GB * N_GB + (gb1 as usize) * N_GB + (gb2 as usize)
+}
+
+/// Determines if there is a grapheme break between two codepoints. This
+/// must be called sequentially maintaining the state between calls.
+///
+/// This function does NOT work with control characters. Control characters,
+/// line feeds, and carriage returns are expected to be filtered out before
+/// calling this function. This is because this function is tuned for
+/// Ghostty.
+#[inline]
+pub fn grapheme_break(cp1: u32, cp2: u32, state: &mut BreakState) -> bool {
+    let t = table();
+    grapheme_break_props(t.get(cp1).grapheme_break, t.get(cp2).grapheme_break, state)
+}
+
+/// Same as [`grapheme_break`] but with already-looked-up break properties.
+#[inline]
+pub fn grapheme_break_props(gb1: Gb, gb2: Gb, state: &mut BreakState) -> bool {
+    let value = PRECOMPUTE[index(gb1, gb2, *state)];
+    *state = value.state;
+    value.result
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::path::Path;
+
+    #[test]
+    fn grapheme_break_emoji_modifier() {
+        // Emoji and modifier
+        {
+            let mut state = BreakState::Default;
+            assert!(!grapheme_break(0x261D, 0x1F3FF, &mut state));
+        }
+
+        // Non-emoji and emoji modifier
+        {
+            let mut state = BreakState::Default;
+            assert!(grapheme_break(0x22, 0x1F3FF, &mut state));
+        }
+    }
+
+    #[test]
+    fn long_emoji_zwj_sequences() {
+        let mut state = BreakState::Default;
+        // (family: woman, woman, girl, boy)
+        let cps: Vec<u32> = "\u{1F469}\u{200D}\u{1F469}\u{200D}\u{1F467}\u{200D}\u{1F466}_"
+            .chars()
+            .map(|c| c as u32)
+            .collect();
+        assert_eq!(cps[0], 0x1F469);
+        for (i, pair) in cps.windows(2).enumerate() {
+            let brk = grapheme_break(pair[0], pair[1], &mut state);
+            // Only the break between the last emoji and `_` is a break.
+            assert_eq!(brk, i == 6, "index {i}");
+        }
+    }
+
+    fn breaks(s: &str) -> Vec<&str> {
+        let mut out = Vec::new();
+        let mut state = BreakState::Default;
+        let mut start = 0;
+        let mut it = s.char_indices().peekable();
+        while let Some((i, c)) = it.next() {
+            let end = i + c.len_utf8();
+            match it.peek() {
+                None => {
+                    out.push(&s[start..end]);
+                }
+                Some(&(_, n)) => {
+                    if grapheme_break(c as u32, n as u32, &mut state) {
+                        out.push(&s[start..end]);
+                        start = end;
+                    }
+                }
+            }
+        }
+        out
+    }
+
+    #[test]
+    fn grapheme_clusters() {
+        assert_eq!(breaks("abc"), ["a", "b", "c"]);
+        // Combining mark
+        assert_eq!(breaks("e\u{301}x"), ["e\u{301}", "x"]);
+        // VS15 / VS16
+        assert_eq!(breaks("\u{2764}\u{FE0F}x"), ["\u{2764}\u{FE0F}", "x"]);
+        assert_eq!(breaks("\u{2764}\u{FE0E}x"), ["\u{2764}\u{FE0E}", "x"]);
+        // Keycap
+        assert_eq!(breaks("1\u{FE0F}\u{20E3}2"), ["1\u{FE0F}\u{20E3}", "2"]);
+        // Skin tone and ZWJ with rocket
+        assert_eq!(breaks("👩🏽‍🚀🇨🇭👨🏻‍🍼"), ["👩🏽‍🚀", "🇨🇭", "👨🏻‍🍼"]);
+        // Sequence of regional indicators: pairs.
+        assert_eq!(
+            breaks("\u{1F1FA}\u{1F1F8}\u{1F1E6}\u{1F1F9}\u{1F1FC}_\u{1F1F3}_"),
+            [
+                "\u{1F1FA}\u{1F1F8}",
+                "\u{1F1E6}\u{1F1F9}",
+                "\u{1F1FC}",
+                "_",
+                "\u{1F1F3}",
+                "_"
+            ]
+        );
+        // Kiss: man, man, light skin tone, dark skin tone (with modifiers).
+        assert_eq!(
+            breaks(
+                "\u{1F468}\u{1F3FB}\u{200D}\u{2764}\u{FE0F}\u{200D}\u{1F48B}\u{200D}\u{1F468}\u{1F3FF}_"
+            ),
+            [
+                "\u{1F468}\u{1F3FB}\u{200D}\u{2764}\u{FE0F}\u{200D}\u{1F48B}\u{200D}\u{1F468}\u{1F3FF}",
+                "_"
+            ]
+        );
+        // Lone emoji modifier after non-base breaks.
+        assert_eq!(breaks("a\u{1F3FB}"), ["a", "\u{1F3FB}"]);
+        // Hangul syllable sequence L V T
+        assert_eq!(
+            breaks("\u{1100}\u{1161}\u{11A8}"),
+            ["\u{1100}\u{1161}\u{11A8}"]
+        );
+        // Prepend
+        assert_eq!(breaks("\u{0600}a"), ["\u{0600}a"]);
+        // Devanagari conjunct (GB9c): KA + virama + SSA
+        assert_eq!(
+            breaks("\u{0915}\u{094D}\u{0937}"),
+            ["\u{0915}\u{094D}\u{0937}"]
+        );
+        // Devanagari no virama: break
+        assert_eq!(breaks("\u{0915}\u{0937}"), ["\u{0915}", "\u{0937}"]);
+    }
+
+    /// Runs the official GraphemeBreakTest.txt (with controls skipped and the
+    /// emoji-modifier tailoring applied, as in uucode's own test).
+    #[test]
+    fn grapheme_break_test_txt() {
+        let path =
+            Path::new(env!("CARGO_MANIFEST_DIR")).join("ucd/auxiliary/GraphemeBreakTest.txt");
+        let content = std::fs::read_to_string(path).unwrap();
+        let t = table();
+        let mut checked = 0;
+        'lines: for (ln, line) in content.lines().enumerate() {
+            let line = line.split('#').next().unwrap().trim();
+            if line.is_empty() {
+                continue;
+            }
+            let mut toks = line.split(' ');
+            assert_eq!(toks.next(), Some("÷"));
+            // Tokens: cp op cp op cp ... op(÷)
+            let mut cps = Vec::new();
+            let mut ops = Vec::new();
+            while let Some(cp) = toks.next() {
+                cps.push(u32::from_str_radix(cp, 16).unwrap());
+                ops.push(toks.next().unwrap());
+            }
+            assert_eq!(*ops.last().unwrap(), "÷");
+            // Skip lines with control/CR/LF (their break property was folded
+            // into `other`; handled by the caller in Ghostty).
+            for &cp in &cps {
+                if cp == 0x0D
+                    || cp == 0x0A
+                    || (cp < 0x20)
+                    || (0x7F..0xA0).contains(&cp)
+                    || [0x2028, 0x2029].contains(&cp)
+                    || is_gb_control(cp)
+                {
+                    continue 'lines;
+                }
+            }
+            let mut state = BreakState::Default;
+            for i in 0..cps.len() - 1 {
+                let gb1 = t.get(cps[i]).grapheme_break;
+                let gb2 = t.get(cps[i + 1]).grapheme_break;
+                let mut expected = ops[i] == "÷";
+                // Ghostty/uucode diverge from UAX #29 for emoji modifiers.
+                if gb2 == Gb::EmojiModifier && gb1 != Gb::EmojiModifierBase {
+                    assert!(!expected);
+                    expected = true;
+                }
+                let actual = grapheme_break(cps[i], cps[i + 1], &mut state);
+                assert_eq!(actual, expected, "line {} idx {i}: {line}", ln + 1);
+                checked += 1;
+            }
+        }
+        assert!(checked > 800, "checked {checked}");
+    }
+
+    /// Codepoints whose original Grapheme_Cluster_Break is Control.
+    fn is_gb_control(cp: u32) -> bool {
+        // Cc, Zl/Zp, and format characters (excluding ZWNJ/ZWJ and prepend)
+        // that are `Control` in GraphemeBreakProperty.txt.
+        matches!(cp, 0x00AD | 0x061C | 0x180E | 0x200B | 0x200E | 0x200F | 0x202A..=0x202E | 0x2060..=0x206F | 0xFEFF | 0xFFF0..=0xFFFB | 0x13430..=0x1343F | 0x1BCA0..=0x1BCA3 | 0x1D173..=0x1D17A | 0xE0000..=0xE001F | 0xE0080..=0xE00FF | 0xE01F0..=0xE0FFF)
+    }
+
+    #[test]
+    fn precompute_matches_compute() {
+        for &s in &ALL_STATES {
+            for &a in &ALL_GB {
+                for &b in &ALL_GB {
+                    let mut s1 = s;
+                    let r1 = compute_grapheme_break_no_control(a, b, &mut s1);
+                    let mut s2 = s;
+                    let r2 = grapheme_break_props(a, b, &mut s2);
+                    assert_eq!((r1, s1), (r2, s2));
+                }
+            }
+        }
+    }
+}

crates/ghostty-unicode/src/lib.rs

@@ -1 +1,184 @@
-//! ghostty-unicode: see docs/rewrite/ARCHITECTURE.md for the module map.
+//! Unicode properties for Ghostty: codepoint width, grapheme breaking, and
+//! the "is symbol" table used by the renderer for constraint width.
+//!
+//! Port of `src/unicode/*`. Ghostty derives these from the `uucode` package
+//! configured by `src/build/uucode_config.zig` (Unicode 17.0.0); here the
+//! same values are generated at build time (`build.rs`) from vendored UCD
+//! files. See `docs/rewrite/notes/ghostty-unicode.md`.
+
+#![cfg_attr(test, allow(clippy::unwrap_used))]
+
+pub mod emoji_table;
+pub mod grapheme;
+pub mod lut;
+pub mod props;
+pub mod props_table;
+pub mod symbols_table;
+
+#[cfg(test)]
+#[path = "../build_support/ucd.rs"]
+mod props_uucode;
+
+pub use grapheme::{BreakState, grapheme_break};
+pub use props::{GraphemeBreakNoControl, Properties};
+pub use props_table::table;
+
+/// Unicode version of the vendored data.
+pub const UNICODE_VERSION: (u8, u8, u8) = (17, 0, 0);
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn unicode_props_tables_match_uucode() {
+        let ucd =
+            props_uucode::Ucd::load(&std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("ucd"));
+        for cp in 0..=0x1F_FFFFu32 {
+            assert_eq!(table().get(cp), ucd.get(cp), "mismatch cp=U+{cp:x}");
+        }
+    }
+
+    #[test]
+    fn unicode_symbols_tables_match_uucode() {
+        let ucd =
+            props_uucode::Ucd::load(&std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("ucd"));
+        for cp in 0..=0x1F_FFFFu32 {
+            assert_eq!(
+                symbols_table::table().get(cp),
+                ucd.is_symbol(cp),
+                "mismatch cp=U+{cp:x}"
+            );
+            assert_eq!(emoji_table::table().get(cp), ucd.is_emoji_presentation(cp));
+        }
+    }
+
+    fn w(cp: u32) -> u8 {
+        table().get(cp).width
+    }
+
+    #[test]
+    fn widths_ascii() {
+        for cp in 0x20..0x7F {
+            assert_eq!(w(cp), 1, "{cp:x}");
+        }
+        // Control characters are zero width (handled before lookup by the
+        // terminal).
+        for cp in (0..0x20).chain(0x7F..0xA0) {
+            assert_eq!(w(cp), 0, "{cp:x}");
+        }
+    }
+
+    #[test]
+    fn widths_cjk_and_fullwidth() {
+        assert_eq!(w('中' as u32), 2);
+        assert_eq!(w('あ' as u32), 2);
+        assert_eq!(w('한' as u32), 2);
+        assert_eq!(w(0xFF21), 2); // fullwidth A
+        assert_eq!(w(0x3000), 2); // ideographic space
+        assert_eq!(w(0xFF61), 1); // halfwidth ideographic full stop
+        assert_eq!(w(0x20000), 2); // CJK ext B
+        assert_eq!(w(0x3FFFD), 2);
+        // Ambiguous width is narrow.
+        assert_eq!(w(0x00B1), 1);
+        assert_eq!(w(0x2460), 1);
+    }
+
+    #[test]
+    fn widths_emoji() {
+        assert_eq!(w(0x1F600), 2); // grinning face
+        assert_eq!(w(0x1F680), 2);
+        assert_eq!(w(0x2764), 1); // heart, text default
+        assert_eq!(w(0x231A), 2); // watch, emoji presentation
+        assert!(table().get(0x2764).emoji_vs_base);
+        assert!(!table().get(0x1F600).emoji_vs_base);
+        assert!(emoji_table::table().get(0x1F600));
+        assert!(!emoji_table::table().get(0x2764));
+    }
+
+    #[test]
+    fn widths_zero_and_combining() {
+        assert_eq!(w(0x0301), 0); // combining acute
+        assert_eq!(w(0x20DD), 0); // enclosing circle
+        assert_eq!(w(0x200B), 0); // ZWSP (default ignorable)
+        assert_eq!(w(0x200D), 0); // ZWJ
+        assert_eq!(w(0x200C), 0); // ZWNJ
+        assert_eq!(w(0xFE0E), 0); // VS15
+        assert_eq!(w(0xFE0F), 0); // VS16
+        assert_eq!(w(0xE0100), 0); // VS17
+        assert_eq!(w(0x2028), 0); // line separator
+        assert_eq!(w(0xD800), 0); // surrogate
+        assert_eq!(w(0x00AD), 1); // soft hyphen
+        assert_eq!(w(0x2E3A), 2); // two-em dash
+        assert_eq!(w(0x2E3B), 2); // three-em dash clamped to 2
+        for cp in [0x0301, 0x200D, 0xFE0F, 0x1F3FB, 0x1160, 0x11A8] {
+            assert!(table().get(cp).width_zero_in_grapheme, "{cp:x}");
+        }
+        assert!(!table().get('a' as u32).width_zero_in_grapheme);
+        assert!(!table().get(0x1F600).width_zero_in_grapheme);
+    }
+
+    #[test]
+    fn widths_emoji_modifier_prepend_ri_keycap() {
+        // Emoji modifiers display as colour patches on their own.
+        for cp in 0x1F3FB..=0x1F3FF {
+            assert_eq!(w(cp), 2);
+            assert!(table().get(cp).width_zero_in_grapheme);
+            assert_eq!(
+                table().get(cp).grapheme_break,
+                GraphemeBreakNoControl::EmojiModifier
+            );
+        }
+        // Prepend characters keep a width.
+        assert_eq!(w(0x0600), 1); // Arabic number sign (prepend, Cf)
+        assert_eq!(
+            table().get(0x0600).grapheme_break,
+            GraphemeBreakNoControl::Prepend
+        );
+        assert!(table().get(0x0600).width_zero_in_grapheme);
+        assert_eq!(w(0x0D4E), 1); // Malayalam letter dot reph
+        // Regional indicators are wide.
+        assert_eq!(w(0x1F1E6), 2);
+        assert_eq!(w(0x1F1FF), 2);
+        assert_eq!(
+            table().get(0x1F1E6).grapheme_break,
+            GraphemeBreakNoControl::RegionalIndicator
+        );
+        // Combining enclosing keycap is zero width in the table.
+        assert_eq!(w(0x20E3), 0);
+        // Hangul jamo.
+        assert_eq!(w(0x1100), 2); // choseong
+        assert_eq!(w(0x1160), 0); // jungseong, zero (V)
+        assert_eq!(w(0x11A8), 0); // jongseong (T)
+    }
+
+    #[test]
+    fn out_of_range_defaults() {
+        // Beyond the Unicode range (props_uucode.zig `get`).
+        let p = table().get(0x11_0000);
+        assert_eq!(p.width, 1);
+        assert!(p.width_zero_in_grapheme);
+        assert_eq!(p.grapheme_break, GraphemeBreakNoControl::Other);
+        assert!(!p.emoji_vs_base);
+        assert_eq!(table().get(0x1F_FFFF), p);
+    }
+
+    #[test]
+    fn symbols() {
+        let t = symbols_table::table();
+        assert!(t.get(0xE000)); // private use
+        assert!(t.get(0xF0001)); // supplementary private use
+        assert!(t.get(0x2190)); // arrows
+        assert!(t.get(0x2713)); // dingbats
+        assert!(t.get(0x1F600)); // emoticons
+        assert!(t.get(0x2600)); // misc symbols
+        assert!(t.get(0x2460)); // enclosed alphanumerics
+        assert!(t.get(0x1F100)); // enclosed alphanumeric supplement
+        assert!(t.get(0x1F300)); // misc symbols & pictographs
+        assert!(t.get(0x1F680)); // transport & map
+        assert!(!t.get('a' as u32));
+        assert!(!t.get('中' as u32));
+        assert!(!t.get(0x2500)); // box drawing
+        assert!(!t.get(0x11_0000));
+    }
+}

crates/ghostty-unicode/src/lut.rs

@@ -0,0 +1,187 @@
+//! Three-level lookup tables for Unicode properties.
+//!
+//! This whole file is based on the algorithm described here:
+//! <https://here-be-braces.com/fast-lookup-of-unicode-properties/>
+//!
+//! NOTE: this file is also `#[path]`-included by `build.rs`, so it must not
+//! depend on anything else in the crate.
+#![allow(dead_code)]
+#![allow(
+    clippy::unwrap_used,
+    clippy::single_match_else,
+    clippy::checked_conversions,
+    clippy::trivially_copy_pass_by_ref,
+    clippy::expect_used
+)]
+
+use std::collections::HashMap;
+use std::fmt::Write as _;
+
+const BLOCK_SIZE: usize = 256;
+
+/// Number of codepoints covered by the tables (`maxInt(u21) + 1`).
+pub const CODEPOINT_COUNT: usize = 0x20_0000;
+
+/// Errors from table generation.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum GenerateError {
+    BlockTooLarge,
+    Stage2TooLarge,
+}
+
+/// Owned lookup tables, the output of [`generate`].
+#[derive(Debug, Clone)]
+pub struct OwnedTables<E> {
+    pub stage1: Vec<u16>,
+    pub stage2: Vec<u16>,
+    pub stage3: Vec<E>,
+}
+
+/// Generate the three-level lookup table for all `u21` codepoints.
+/// `get` returns the mapping for a codepoint; mappings are compared with `==`
+/// (the Zig `Context.eql`).
+pub fn generate<E: Clone + PartialEq>(
+    mut get: impl FnMut(u32) -> E,
+) -> Result<OwnedTables<E>, GenerateError> {
+    let mut blocks_map: HashMap<Vec<u16>, u16> = HashMap::new();
+    let mut stage1: Vec<u16> = Vec::new();
+    let mut stage2: Vec<u16> = Vec::new();
+    let mut stage3: Vec<E> = Vec::new();
+
+    let mut block: Vec<u16> = Vec::with_capacity(BLOCK_SIZE);
+    for cp in 0..CODEPOINT_COUNT {
+        // Get our block value and find the matching result value in our list
+        // of possible values in stage3. This way, each possible mapping only
+        // gets one entry in stage3.
+        let elem = get(cp as u32);
+        let idx = match stage3.iter().position(|item| *item == elem) {
+            Some(i) => i,
+            None => {
+                stage3.push(elem);
+                stage3.len() - 1
+            }
+        };
+        block.push(u16::try_from(idx).map_err(|_| GenerateError::BlockTooLarge)?);
+
+        if block.len() < BLOCK_SIZE && cp != CODEPOINT_COUNT - 1 {
+            continue;
+        }
+        block.resize(BLOCK_SIZE, 0);
+
+        let next = stage2.len();
+        let s2 = match blocks_map.get(&block) {
+            Some(&s) => s,
+            None => {
+                let s = u16::try_from(next).map_err(|_| GenerateError::Stage2TooLarge)?;
+                stage2.extend_from_slice(&block);
+                blocks_map.insert(block.clone(), s);
+                s
+            }
+        };
+        stage1.push(s2);
+        block.clear();
+    }
+
+    assert!(stage1.len() <= usize::from(u16::MAX));
+    assert!(stage2.len() <= usize::from(u16::MAX));
+    assert!(stage3.len() <= usize::from(u16::MAX));
+    Ok(OwnedTables {
+        stage1,
+        stage2,
+        stage3,
+    })
+}
+
+impl<E> OwnedTables<E> {
+    /// Borrow as lookup [`Tables`].
+    #[must_use]
+    pub fn as_tables(&self) -> Tables<'_, E> {
+        Tables {
+            stage1: &self.stage1,
+            stage2: &self.stage2,
+            stage3: &self.stage3,
+        }
+    }
+
+    /// Writes the lookup table as Rust source. The output defines three
+    /// statics (`STAGE1`, `STAGE2`, `STAGE3`) that can be used to rebuild the
+    /// lookup table.
+    pub fn write_rust(&self, elem_ty: &str, mut fmt_elem: impl FnMut(&E) -> String) -> String {
+        let mut out = String::new();
+        out.push_str("// This file is auto-generated. Do not edit.\n\n");
+        let _ = write!(out, "pub static STAGE1: [u16; {}] = [", self.stage1.len());
+        for e in &self.stage1 {
+            let _ = write!(out, "{e},");
+        }
+        out.push_str("];\n\n");
+        let _ = write!(out, "pub static STAGE2: [u16; {}] = [", self.stage2.len());
+        for e in &self.stage2 {
+            let _ = write!(out, "{e},");
+        }
+        out.push_str("];\n\n");
+        let _ = write!(
+            out,
+            "pub static STAGE3: [{elem_ty}; {}] = [",
+            self.stage3.len()
+        );
+        for e in &self.stage3 {
+            let _ = write!(out, "{},", fmt_elem(e));
+        }
+        out.push_str("];\n");
+        out
+    }
+}
+
+/// A 3-level lookup table from a Unicode codepoint to a mapping of type `E`.
+#[derive(Debug, Clone, Copy)]
+pub struct Tables<'a, E> {
+    pub stage1: &'a [u16],
+    pub stage2: &'a [u16],
+    pub stage3: &'a [E],
+}
+
+impl<E: Copy> Tables<'_, E> {
+    /// Given a codepoint (at most `0x1F_FFFF`), returns the mapping for it.
+    #[inline]
+    #[must_use]
+    pub fn get(&self, cp: u32) -> E {
+        let high = (cp >> 8) as usize;
+        let low = (cp & 0xFF) as usize;
+        self.stage3[self.stage2[self.stage1[high] as usize + low] as usize]
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn generate_roundtrip() {
+        let f = |cp: u32| -> u8 {
+            if cp < 0x80 {
+                0
+            } else if (0x1000..0x1100).contains(&cp) {
+                1
+            } else if cp == 0x1F600 {
+                2
+            } else {
+                3
+            }
+        };
+        let t = generate(f).unwrap();
+        let tables = t.as_tables();
+        for cp in 0..CODEPOINT_COUNT as u32 {
+            assert_eq!(tables.get(cp), f(cp), "cp={cp:x}");
+        }
+        // Blocks are deduplicated.
+        assert!(t.stage2.len() < 10 * BLOCK_SIZE);
+        assert_eq!(t.stage1.len(), CODEPOINT_COUNT / BLOCK_SIZE);
+    }
+
+    #[test]
+    fn write_rust_output() {
+        let t = generate(|cp| cp == 0x41).unwrap();
+        let src = t.write_rust("bool", ToString::to_string);
+        assert!(src.contains("pub static STAGE3: [bool; 2] = [false,true,];"));
+    }
+}

crates/ghostty-unicode/src/props.rs

@@ -0,0 +1,100 @@
+//! Property set per codepoint that Ghostty cares about.
+//!
+//! Adding to this lets you find new properties but also potentially makes
+//! our lookup tables less efficient.
+//!
+//! NOTE: this file is also `#[path]`-included by `build.rs`, so it must not
+//! depend on anything else in the crate.
+
+/// Grapheme break property with the control/CR/LF classes removed
+/// (`uucode.x.types.GraphemeBreakNoControl`). The discriminant order matches
+/// uucode and is used to index the precomputed break table.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
+#[repr(u8)]
+pub enum GraphemeBreakNoControl {
+    #[default]
+    Other = 0,
+    Prepend,
+    RegionalIndicator,
+    SpacingMark,
+    L,
+    V,
+    T,
+    Lv,
+    Lvt,
+    Zwj,
+    Zwnj,
+    ExtendedPictographic,
+    EmojiModifierBase,
+    EmojiModifier,
+    // extend, ==
+    //   zwnj +
+    //   indic_conjunct_break_extend +
+    //   indic_conjunct_break_linker
+    IndicConjunctBreakExtend,
+    IndicConjunctBreakLinker,
+    IndicConjunctBreakConsonant,
+}
+
+impl GraphemeBreakNoControl {
+    /// Number of variants.
+    pub const COUNT: usize = 17;
+
+    /// Variant name in Rust syntax (used by the table generator).
+    #[must_use]
+    pub const fn rust_name(self) -> &'static str {
+        match self {
+            Self::Other => "Other",
+            Self::Prepend => "Prepend",
+            Self::RegionalIndicator => "RegionalIndicator",
+            Self::SpacingMark => "SpacingMark",
+            Self::L => "L",
+            Self::V => "V",
+            Self::T => "T",
+            Self::Lv => "Lv",
+            Self::Lvt => "Lvt",
+            Self::Zwj => "Zwj",
+            Self::Zwnj => "Zwnj",
+            Self::ExtendedPictographic => "ExtendedPictographic",
+            Self::EmojiModifierBase => "EmojiModifierBase",
+            Self::EmojiModifier => "EmojiModifier",
+            Self::IndicConjunctBreakExtend => "IndicConjunctBreakExtend",
+            Self::IndicConjunctBreakLinker => "IndicConjunctBreakLinker",
+            Self::IndicConjunctBreakConsonant => "IndicConjunctBreakConsonant",
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
+pub struct Properties {
+    /// Codepoint width. We clamp to [0, 2] since Ghostty handles control
+    /// characters and we max out at 2 for wide characters (i.e. 3-em dash
+    /// becomes a 2-em dash). (`u2` in Zig.)
+    pub width: u8,
+
+    /// Whether the code point does not contribute to the width of a grapheme
+    /// cluster (not used for single code point cells).
+    pub width_zero_in_grapheme: bool,
+
+    /// Grapheme break property.
+    pub grapheme_break: GraphemeBreakNoControl,
+
+    /// Emoji VS compatibility
+    pub emoji_vs_base: bool,
+}
+
+impl Properties {
+    /// Rust source expression for this value (the Zig `format`), used by the
+    /// build script when writing the generated tables.
+    #[must_use]
+    #[allow(clippy::trivially_copy_pass_by_ref)]
+    pub fn format_rust(&self) -> String {
+        format!(
+            "Properties {{ width: {}, width_zero_in_grapheme: {}, grapheme_break: GraphemeBreakNoControl::{}, emoji_vs_base: {} }}",
+            self.width,
+            self.width_zero_in_grapheme,
+            self.grapheme_break.rust_name(),
+            self.emoji_vs_base,
+        )
+    }
+}

crates/ghostty-unicode/src/props_table.rs

@@ -0,0 +1,21 @@
+//! The generated Unicode property lookup table for Ghostty
+//! (`props_table.zig`). Generated by `build.rs`.
+
+use crate::lut::Tables;
+use crate::props::{GraphemeBreakNoControl, Properties};
+
+mod generated {
+    use super::{GraphemeBreakNoControl, Properties};
+    include!(concat!(env!("OUT_DIR"), "/props_tables.rs"));
+}
+
+/// The lookup tables for Ghostty.
+#[inline]
+#[must_use]
+pub fn table() -> Tables<'static, Properties> {
+    Tables {
+        stage1: &generated::STAGE1,
+        stage2: &generated::STAGE2,
+        stage3: &generated::STAGE3,
+    }
+}

crates/ghostty-unicode/src/symbols_table.rs

@@ -0,0 +1,19 @@
+//! The generated `is_symbol` lookup table (used by the renderer for constraint width) (`symbols_table.zig`). Generated by
+//! `build.rs`.
+
+use crate::lut::Tables;
+
+mod generated {
+    include!(concat!(env!("OUT_DIR"), "/symbols_tables.rs"));
+}
+
+/// The lookup table (codepoint -> bool).
+#[inline]
+#[must_use]
+pub fn table() -> Tables<'static, bool> {
+    Tables {
+        stage1: &generated::STAGE1,
+        stage2: &generated::STAGE2,
+        stage3: &generated::STAGE3,
+    }
+}