feat(vt): port the stream layer between parser and handlers

Ports ZIG/src/terminal/stream.zig: the Stream feeds bytes through the
UTF-8 decoder and parser and dispatches every CSI/ESC/OSC/DCS/APC and
C0/C1 control to a Handler as a high-level Action (all 95 Zig
variants). Zig's comptime handler interface becomes a Handler trait with
an associated error type, a required vt() and an optional vt_raw() hook.

Performance: next_slice() processes 4 KiB chunks, finds ESC with memchr
and decodes printable runs in bulk (scalar port of ZIG/src/simd's
index_of / utf8_decode_until_control_seq), plus the CSI parameter fast
path.

Terminals must construct the stream with Stream::new_alloc so the OSC
parser accepts allocating commands (colors, OSC 133), as in Zig.

Minor differences: after a stray UTF-8 byte the rest of the chunk
returns to the fast path, and a spurious Zig warning after CSI 2/5 W is
dropped.

Test Plan:
- cargo test -p ghostty-vt (448 passed; 37/38 stream tests ported,
  the skipped one only exercises the C ABI)

crates/ghostty-vt/src/simd.rs

@@ -0,0 +1,207 @@
+//! SIMD-optimized routines (ZIG/src/simd). The Rust port is scalar-only:
+//! `memchr` provides the vectorized byte search, and UTF-8 decoding follows
+//! the Zig scalar fallback (`utf8DecodeUntilControlSeqScalar`), which is
+//! tested to match the SIMD behavior.
+//!
+//! Not ported: `codepoint_width` (belongs to `ghostty-unicode`) and
+//! `base64` (the `base64` crate is used by `kitty::graphics_command`).
+
+#![allow(clippy::single_match_else)]
+
+/// Find the first occurrence of `needle` in `input`.
+pub fn index_of(input: &[u8], needle: u8) -> Option<usize> {
+    memchr::memchr(needle, input)
+}
+
+/// Result of [`utf8_decode_until_control_seq`].
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct DecodeResult {
+    /// Number of input bytes consumed.
+    pub consumed: usize,
+    /// Number of codepoints written to the output.
+    pub decoded: usize,
+}
+
+/// Length of a UTF-8 sequence given its leading byte, `None` if the byte
+/// cannot start a sequence (Zig `std.unicode.utf8ByteSequenceLength`).
+fn utf8_byte_sequence_length(first: u8) -> Option<usize> {
+    match first {
+        0x00..=0x7F => Some(1),
+        0xC0..=0xDF => Some(2),
+        0xE0..=0xEF => Some(3),
+        0xF0..=0xF7 => Some(4),
+        _ => None,
+    }
+}
+
+/// Decode `bytes` (exactly one sequence) strictly; `None` if ill-formed.
+fn utf8_decode(bytes: &[u8]) -> Option<u32> {
+    std::str::from_utf8(bytes)
+        .ok()
+        .and_then(|s| s.chars().next())
+        .map(|c| c as u32)
+}
+
+/// Decode UTF-8 from `input` into `output` until an ESC (0x1B) is found.
+/// `output.len()` must be at least `input.len()`.
+///
+/// Invalid sequences have their leading byte replaced with U+FFFD and
+/// decoding continues after that byte. Decoding stops (without consuming)
+/// at an incomplete trailing sequence or at a byte that can't start a
+/// sequence (e.g. 0xFF or a stray continuation byte); the caller must handle
+/// those bytes itself (see `stream.rs`).
+pub fn utf8_decode_until_control_seq(input: &[u8], output: &mut [u32]) -> DecodeResult {
+    assert!(output.len() >= input.len());
+
+    let idx = index_of(input, 0x1B).unwrap_or(input.len());
+    let decode = &input[..idx];
+
+    let mut decode_offset = 0;
+    let mut decode_count = 0;
+    while decode_offset < decode.len() {
+        let rem = &decode[decode_offset..];
+        let Some(cp_len) = utf8_byte_sequence_length(rem[0]) else {
+            break;
+        };
+
+        // Partial input: defer to the future.
+        if rem.len() < cp_len {
+            break;
+        }
+
+        match utf8_decode(&rem[..cp_len]) {
+            Some(cp) => {
+                output[decode_count] = cp;
+                decode_offset += cp_len;
+            }
+            None => {
+                // Replace only the leading byte and continue after it.
+                output[decode_count] = 0xFFFD;
+                decode_offset += 1;
+            }
+        }
+        decode_count += 1;
+    }
+
+    DecodeResult {
+        consumed: decode_offset,
+        decoded: decode_count,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn index_of_basic() {
+        assert_eq!(index_of(b"hello", b' '), None);
+        assert_eq!(index_of(b"hi lo", b' '), Some(2));
+        assert_eq!(
+            index_of(
+                b"XXXXX XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\nXXXXXXXXXXXX XXXXXXXXXXX XXXXXXXXXXXXXXX",
+                b' '
+            ),
+            Some(5)
+        );
+        assert_eq!(
+            index_of(
+                b"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\nXXXXXXXXXXXX XXXXXXXXXXX XXXXXXXXXXXXXXX",
+                b' '
+            ),
+            Some(53)
+        );
+    }
+
+    #[test]
+    fn decode_no_escape() {
+        let mut output = [0u32; 1024];
+        let s = "hello".repeat(128);
+        assert_eq!(
+            utf8_decode_until_control_seq(s.as_bytes(), &mut output),
+            DecodeResult {
+                consumed: s.len(),
+                decoded: s.len()
+            }
+        );
+    }
+
+    #[test]
+    fn decode_ascii_to_escape() {
+        let mut output = [0u32; 1024];
+        let prefix = "hello".repeat(64);
+        let s = format!("{prefix}\x1bworld");
+        assert_eq!(
+            utf8_decode_until_control_seq(s.as_bytes(), &mut output),
+            DecodeResult {
+                consumed: prefix.len(),
+                decoded: prefix.len()
+            }
+        );
+    }
+
+    #[test]
+    fn decode_immediate_esc_sequence() {
+        let mut output = [0u32; 64];
+        assert_eq!(
+            utf8_decode_until_control_seq(b"\x1b[?5s", &mut output),
+            DecodeResult {
+                consumed: 0,
+                decoded: 0
+            }
+        );
+    }
+
+    #[test]
+    fn decode_incomplete_utf8() {
+        let mut output = [0u32; 64];
+        for s in [
+            &b"hello\xc2"[..],
+            &b"hello\xe0\x00"[..],
+            &b"hello\xf0\x90"[..],
+        ] {
+            assert_eq!(
+                utf8_decode_until_control_seq(s, &mut output),
+                DecodeResult {
+                    consumed: 5,
+                    decoded: 5
+                }
+            );
+        }
+    }
+
+    #[test]
+    fn decode_invalid_utf8() {
+        let mut output = [0u32; 64];
+        assert_eq!(
+            utf8_decode_until_control_seq(b"hello\xc2\x01", &mut output),
+            DecodeResult {
+                consumed: 7,
+                decoded: 7
+            }
+        );
+        // Replacement only replaces the invalid leading byte.
+        assert_eq!(output[5], 0xFFFD);
+        assert_eq!(output[6], 0x01);
+    }
+
+    #[test]
+    fn decode_invalid_leading_byte_isnt_consumed_or_replaced() {
+        let mut output = [0u32; 64];
+        assert_eq!(
+            utf8_decode_until_control_seq(b"hello\xFF", &mut output),
+            DecodeResult {
+                consumed: 5,
+                decoded: 5
+            }
+        );
+    }
+
+    #[test]
+    fn decode_multibyte() {
+        let mut output = [0u32; 64];
+        let r = utf8_decode_until_control_seq("a\u{e9}\u{4e00}\u{1f600}".as_bytes(), &mut output);
+        assert_eq!(r.decoded, 4);
+        assert_eq!(&output[..4], &[0x61, 0xE9, 0x4E00, 0x1F600]);
+    }
+}