test: differential fuzzing of the terminal core against Ghostty's Zig code

Adds tools/difftest: a Zig driver built against a copy of Ghostty
1.3.1's own src/terminal (Zig 0.15.2, set up under target/ by
tools/difftest/setup.sh) and a Rust twin on ghostty-terminal, both
printing a canonical dump of every scrollback/active row (cells,
graphemes, styles, hyperlinks, row flags), cursor, modes, regions,
tabstops and colors. A seeded generator produces escape-heavy streams
(UTF-8/wide/combining/emoji, C0, CSI with edge params, SGR, modes, ESC,
OSC, DCS, margins, insert/delete); `just difftest` runs it with Zig
assertions on (ReleaseSafe vs debug Rust) and off (ReleaseFast vs
production Rust), with minimizers and a triage script.

Divergences found and fixed in Rust (each with a regression test):
- ill-formed UTF-8 decoding follows the SIMD path a default Ghostty
  build uses (not Zig's scalar fallback)
- PageList::erase_rows keeps Zig's row-flag behavior on regrow
- grapheme move on wrap with left/right margins uses the original cell

Upstream bugs found and deliberately NOT replicated (per the project
rule): a stale cell pointer after page growth in Terminal.zig:482/539
that drops a grapheme, plus several Zig assertion failures/segfaults;
documented with minimized repros in docs/rewrite/notes/difftest-DF.md.

Final run: 390k cases across many geometries, 0 divergences (Zig-abort
cases, ~0.1-0.5%, are not compared).

Test Plan:
- just difftest 20000 "1 2 3"
- cargo test -p ghostty-terminal (regression tests)
This commit is contained in:
ddidderr committed 2026-10-12 13:24:12 +02:00
1 parent 6def1f3b5d
commit 4eb03f637d
19 files changed
+2033 -165

No files matched your search

+21 -3
View File
@@ -1094,10 +1094,28 @@ impl PageList {
page.size.rows = 0;
} else {
let page = &mut self.node_mut(id).ok_or(Error::InvalidId)?.data;
for y in a..b {
page.clear_cells(y, 0, usize::from(page.size.cols), false)?;
if a == 0 {
// Zig (`eraseRows`) SWAPS the surviving rows into place
// rather than rotating, then clears the cells of the
// rows left over past the new size. Row metadata (e.g.
// the `wrap` flag) of those leftover rows is NOT reset
// and becomes visible again when the page regrows into
// them, so the exact arrangement must match.
let cols = usize::from(page.size.cols);
let scroll_amount = rows - b;
for i in 0..scroll_amount {
page.rows.swap(i, i + b);
page.rows[i].set_dirty(true);
}
for y in scroll_amount..rows {
page.clear_cells(y, 0, cols, false)?;
}
} else {
for y in a..b {
page.clear_cells(y, 0, usize::from(page.size.cols), false)?;
}
page.rows[a..rows].rotate_left(b - a);
}
page.rows[a..rows].rotate_left(b - a);
page.size.rows -= (b - a) as u16;
for row in &mut page.rows[a..usize::from(page.size.rows)] {
row.set_dirty(true);
@@ -1022,4 +1022,94 @@ mod tests {
s.next_slice(b"\x1b[?W").unwrap();
}
// ---- Regressions found by the Zig-vs-Rust differential fuzzer
// (tools/difftest, docs/rewrite/notes/difftest-DF.md). Expected values
// come from the Zig reference driver.
fn screen_row_wrap(t: &Terminal, y: u32) -> bool {
use crate::point::{Coordinate, Point};
let pages = &t.screen().pages;
let pin = pages
.pin(Point::Screen(Coordinate { x: 0, y }))
.expect("row exists");
pin.row_and_cell(pages).expect("row").0.wrap()
}
fn cell_cps(t: &Terminal, row: u32) -> Vec<u32> {
use crate::point::{Coordinate, Point};
(0..t.cols)
.map(|x| {
t.screen()
.pages
.get_cell(Point::Active(Coordinate { x, y: row }))
.map_or(0, crate::page::Cell::codepoint)
})
.collect()
}
#[test]
fn difftest_invalid_utf8_overlong_swallows_trail_byte() {
// Zig's SIMD path (utfcpp replace_invalid) emits ONE U+FFFD for an
// overlong lead plus its continuation byte.
let mut t = term(10, 2);
t.vt_stream().next_slice(b"\xc0\xa9").unwrap();
assert_eq!(1, t.screen().cursor.x);
assert_eq!(0xFFFD, cell_cps(&t, 0)[0]);
}
#[test]
fn difftest_invalid_utf8_truncated_then_control() {
// e4 a9 followed by a non-trail byte: the truncated sequence is a
// single replacement and the following byte decodes normally.
let mut t = term(10, 2);
t.vt_stream().next_slice(b"\xe4\xa9A\xf0\x9f").unwrap();
let cps = cell_cps(&t, 0);
assert_eq!(&cps[..2], &[0xFFFD, 'A' as u32]);
}
#[test]
fn difftest_erase_scrollback_keeps_stale_row_flags() {
// Zig's eraseRows swaps rows and leaves the stale `wrap` flag in the
// leftover rows, which is visible once the page regrows.
let mut t = term(1, 1);
t.vt_stream().next_slice(b"xt\n\x1b[3Jex").unwrap();
assert!(screen_row_wrap(&t, 0));
assert!(!screen_row_wrap(&t, 1));
}
#[test]
fn difftest_grapheme_wrap_with_margins_moves_prev_cell() {
// Devanagari conjunct widening the cell at the last column while a
// left/right margin keeps the wrap from scrolling that cell: Zig moves
// the grapheme from the original cell pointer (not `up(1)`).
use crate::point::{Coordinate, Point};
let mut t = term(20, 8);
t.vt_stream()
.next_slice("\x1b[?2027h\x1b[?69h\x1b[;10s\x1b[50;41H\u{915}\u{94d}\u{937}".as_bytes())
.unwrap();
let pages = &t.screen().pages;
let pin = pages.pin(Point::Active(Coordinate { x: 0, y: 7 })).unwrap();
let (_, cell) = pin.row_and_cell(pages).unwrap();
assert_eq!(cell.codepoint(), 0x915);
assert_eq!(crate::page::Wide::Wide, cell.wide());
assert_eq!(Some(&[0x94d, 0x937][..]), pin.grapheme(pages));
}
/// Upstream (Zig) bug, deliberately NOT replicated: Zig's
/// `Terminal.print` grapheme path keeps `prev.cell` across a page
/// reallocation (hyperlink/string capacity growth) and appends the
/// grapheme into freed memory, silently losing U+FE0F here. Rust
/// addresses cells by index and keeps it. See difftest-DF.md.
#[test]
fn difftest_upstream_uaf_grapheme_survives_page_growth() {
let mut t = Terminal::new(Options {
max_scrollback: 1,
..Options::new(5, 30)
})
.unwrap();
let input = b"\x1b\x5d\x38\x3b\x3b\x6c\x07\x35\x36\x37\x38\x39\x1b\x5b\x38\x65\x5f\x2c\xba\x9f\x87\xb8\x80\xbc\xbb\x1b\x5b\x35\x32\x72\x54\x68\x65\x20\x71\x75\x69\x63\x6b\x20\x62\x72\x6f\x77\x6e\x20\x66\x6f\x78\x1b\x5c\x61\x20\x40\x1b\x5b\x53\x1b\x4b\x30\x31\x32\x33\x34\x35\x36\x37\x31\x53\x1b\x73\x5d\x5b\x32\x6d\x9b\xff\x80\x80\x1b\x5b\x3f\x32\x30\x32\x37\x68\x1b\x5b\x3b\x33\x48\xf0\xc0\x5c\x77\x70\x5e\x6f\x20\x70\xe2\x9d\xa4\xef\xb8\x8f";
t.vt_stream().next_slice(input).unwrap();
assert!(t.plain_string().unwrap().contains("\u{2764}\u{fe0f}"));
}
}
+15 -1
View File
@@ -904,6 +904,17 @@ impl Terminal {
// This is like print_cell but without clearing the
// grapheme data from the cell, so we can move it
// later.
// Zig keeps a raw pointer (`prev.cell`) to this cell;
// remember its page cell index, because the wrap below
// may scroll (rows move, cell memory does not) and the
// grapheme move must still use THIS cell.
let prev_idx = {
let scr = self.screen_mut();
let (x, pin) = (scr.cursor.x, scr.cursor_pin());
scr.page(pin.node)
.cell_index(usize::from(x), usize::from(pin.y))
.expect("prev cell")
};
{
let cell = self.screen_mut().cursor_cell_mut();
cell.set_wide(if row_wrap {
@@ -929,8 +940,11 @@ impl Terminal {
if new_pin.node == old_pin.node {
let page = self.screen_mut().page_mut(new_pin.node);
let old_idx = page.cell_index(old_x, old_y).expect("old cell");
let new_idx = page.cell_index(new_x, new_y).expect("new cell");
// Zig moves from `prev.cell`, NOT from `old_pin`
// (they differ when margins keep the row from
// scrolling the cell out of `old_pin`).
let old_idx = prev_idx;
page.move_grapheme(old_idx, new_idx)?;
page.cells[old_idx].set_content_tag(ContentTag::Codepoint);
page.cells[new_idx].set_content_tag(ContentTag::CodepointGrapheme);
+179 -135
View File
@@ -1,7 +1,10 @@
//! SIMD-optimized routines (ZIG/src/simd). The Rust port is scalar-only:
//! `memchr` provides the vectorized byte search, and UTF-8 decoding follows
//! the Zig scalar fallback (`utf8DecodeUntilControlSeqScalar`), which is
//! tested to match the SIMD behavior.
//! `memchr` provides the vectorized byte search, and UTF-8 decoding
//! reproduces the observable behaviour of the C++ SIMD path (`vt.cpp`:
//! simdutf + utfcpp `replace_invalid` + `trim_partial_utf8`), which is what a
//! default Ghostty build runs. (It differs from Zig's scalar fallback
//! `utf8DecodeUntilControlSeqScalar` for ill-formed input; found by the
//! differential fuzzer, see docs/rewrite/notes/difftest-DF.md.)
//!
//! Not ported: `codepoint_width` (belongs to `ghostty-unicode`) and
//! `base64` (the `base64` crate is used by `kitty::graphics_command`).
@@ -22,113 +25,155 @@ pub struct DecodeResult {
pub decoded: usize,
}
/// Length of a UTF-8 sequence given its leading byte, `None` if the byte
/// cannot start a sequence (Zig `std.unicode.utf8ByteSequenceLength`).
fn utf8_byte_sequence_length(first: u8) -> Option<usize> {
match first {
0x00..=0x7F => Some(1),
0xC0..=0xDF => Some(2),
0xE0..=0xEF => Some(3),
0xF0..=0xF7 => Some(4),
_ => None,
/// `simdutf::scalar::utf8::trim_partial_utf8`: the length of `input` with a
/// trailing (possibly) incomplete UTF-8 sequence removed. Purely byte-value
/// based (it does not validate), exactly like the C++.
fn trim_partial_utf8(input: &[u8]) -> usize {
let length = input.len();
if length < 3 {
match length {
2 => {
if input[1] >= 0xc0 {
return 1;
}
if input[0] >= 0xe0 {
return 0;
}
return length;
}
1 => {
if input[0] >= 0xc0 {
return 0;
}
return length;
}
_ => return length,
}
}
if input[length - 1] >= 0xc0 {
return length - 1;
}
if input[length - 2] >= 0xe0 {
return length - 2;
}
if input[length - 3] >= 0xf0 {
return length - 3;
}
length
}
/// Decode `bytes` (exactly one sequence whose length matches its leading
/// byte) strictly; `None` if ill-formed (overlong, surrogate, out of range or
/// a bad continuation byte). Equivalent to `std::str::from_utf8` on one char.
#[inline]
fn utf8_decode(bytes: &[u8]) -> Option<u32> {
/// Decode `input` into `output` replacing invalid sequences with U+FFFD the
/// way `DecodeUTF8` in `vt.cpp` does: simdutf for the all-valid fast path and
/// `utf8::replace_invalid` (utfcpp) otherwise. Because replacement output is
/// valid UTF-8, this is equivalent to a single utfcpp pass:
///
/// - an invalid lead byte (`0x80..=0xC1`-style continuation bytes, `0xF8..`)
/// becomes one U+FFFD and only that byte is skipped;
/// - an incomplete sequence, overlong encoding, surrogate or out-of-range
/// code point becomes ONE U+FFFD and the lead byte plus every following
/// continuation byte is skipped;
/// - a sequence truncated by the end of input becomes one U+FFFD.
fn decode_utf8_replace(input: &[u8], output: &mut [u32]) -> usize {
#[inline]
fn cont(b: u8) -> bool {
fn is_trail(b: u8) -> bool {
b & 0xC0 == 0x80
}
match *bytes {
[b0] if b0 < 0x80 => Some(u32::from(b0)),
[b0 @ 0xC2..=0xDF, b1] if cont(b1) => {
Some((u32::from(b0 & 0x1F) << 6) | u32::from(b1 & 0x3F))
let mut count = 0;
let mut i = 0;
while i < input.len() {
let lead = input[i];
let len = if lead < 0x80 {
1
} else if lead >> 5 == 0x6 {
2
} else if lead >> 4 == 0xe {
3
} else if lead >> 3 == 0x1e {
4
} else {
// INVALID_LEAD
output[count] = 0xFFFD;
count += 1;
i += 1;
continue;
};
// Gather the sequence, tracking utfcpp's error classes.
let mut cp = if len == 1 {
u32::from(lead)
} else {
u32::from(lead) & (0x7f >> len)
};
let mut err = None; // Some(true) = NOT_ENOUGH_ROOM, Some(false) = other
for k in 1..len {
match input.get(i + k) {
None => {
err = Some(true);
break;
}
Some(&b) if !is_trail(b) => {
err = Some(false);
break;
}
Some(&b) => cp = (cp << 6) | u32::from(b & 0x3f),
}
}
[b0 @ 0xE0..=0xEF, b1, b2] if cont(b1) && cont(b2) => {
let ok = match b0 {
0xE0 => b1 >= 0xA0,
0xED => b1 < 0xA0,
_ => true,
if err.is_none() {
let valid = cp <= 0x10_FFFF && !(0xD800..=0xDFFF).contains(&cp);
let overlong = match cp {
0..=0x7f => len != 1,
0x80..=0x7ff => len != 2,
0x800..=0xffff => len != 3,
_ => false,
};
ok.then(|| {
(u32::from(b0 & 0x0F) << 12) | (u32::from(b1 & 0x3F) << 6) | u32::from(b2 & 0x3F)
})
if valid && !overlong {
output[count] = cp;
count += 1;
i += len;
continue;
}
err = Some(false);
}
[b0 @ 0xF0..=0xF4, b1, b2, b3] if cont(b1) && cont(b2) && cont(b3) => {
let ok = match b0 {
0xF0 => b1 >= 0x90,
0xF4 => b1 < 0x90,
_ => true,
};
ok.then(|| {
(u32::from(b0 & 0x07) << 18)
| (u32::from(b1 & 0x3F) << 12)
| (u32::from(b2 & 0x3F) << 6)
| u32::from(b3 & 0x3F)
})
output[count] = 0xFFFD;
count += 1;
if err == Some(true) {
// NOT_ENOUGH_ROOM: one replacement for the truncated tail.
break;
}
i += 1;
while i < input.len() && is_trail(input[i]) {
i += 1;
}
_ => None,
}
count
}
/// Decode UTF-8 from `input` into `output` until an ESC (0x1B) is found.
/// `output.len()` must be at least `input.len()`.
/// Decode UTF-8 from `input` into `output` until an ESC (0x1B) is found,
/// faithfully following `ghostty_simd_decode_utf8_until_control_seq`
/// (`ZIG/src/simd/vt.cpp`, the real SIMD path of a default Ghostty build):
///
/// Invalid sequences have their leading byte replaced with U+FFFD and
/// decoding continues after that byte. Decoding stops (without consuming)
/// at an incomplete trailing sequence or at a byte that can't start a
/// sequence (e.g. 0xFF or a stray continuation byte); the caller must handle
/// those bytes itself (see `stream.rs`).
/// - if an ESC is found, everything before it is decoded (invalid and
/// truncated sequences become U+FFFD, see [`decode_utf8_replace`]) and the
/// ESC position is returned;
/// - otherwise a trailing partial sequence is trimmed with
/// `trim_partial_utf8` (a byte-value heuristic, it may trim valid trailing
/// bytes too) and the rest is decoded; the caller feeds the untouched tail
/// to the scalar decoder.
///
/// `output.len()` must be at least `input.len()`.
pub fn utf8_decode_until_control_seq(input: &[u8], output: &mut [u32]) -> DecodeResult {
assert!(output.len() >= input.len());
let idx = index_of(input, 0x1B).unwrap_or(input.len());
let decode = &input[..idx];
let mut decode_offset = 0;
let mut decode_count = 0;
while decode_offset < decode.len() {
// ASCII fast path: a run of single-byte sequences.
let rem = &decode[decode_offset..];
if rem[0] < 0x80 {
let run = rem.iter().take_while(|&&b| b < 0x80).count();
for (o, &b) in output[decode_count..decode_count + run].iter_mut().zip(rem) {
*o = u32::from(b);
}
decode_count += run;
decode_offset += run;
continue;
}
let Some(cp_len) = utf8_byte_sequence_length(rem[0]) else {
break;
if let Some(esc) = index_of(input, 0x1B) {
let decoded = decode_utf8_replace(&input[..esc], output);
return DecodeResult {
consumed: esc,
decoded,
};
// Partial input: defer to the future.
if rem.len() < cp_len {
break;
}
match utf8_decode(&rem[..cp_len]) {
Some(cp) => {
output[decode_count] = cp;
decode_offset += cp_len;
}
None => {
// Replace only the leading byte and continue after it.
output[decode_count] = 0xFFFD;
decode_offset += 1;
}
}
decode_count += 1;
}
let trimmed = trim_partial_utf8(input);
let decoded = decode_utf8_replace(&input[..trimmed], output);
DecodeResult {
consumed: decode_offset,
decoded: decode_count,
consumed: trimmed,
decoded,
}
}
@@ -137,56 +182,55 @@ mod tests {
use super::*;
#[test]
fn utf8_decode_matches_std() {
fn std_decode(bytes: &[u8]) -> Option<u32> {
std::str::from_utf8(bytes)
.ok()
.and_then(|s| s.chars().next())
.map(|c| c as u32)
}
for b0 in 0..=255u8 {
let Some(len) = utf8_byte_sequence_length(b0) else {
fn decode_valid_matches_std() {
let mut out = [0u32; 8];
for cp in (0..=0x10FFFFu32)
.step_by(7)
.chain([0x7f, 0x80, 0x7ff, 0x800, 0xffff, 0x10000, 0x10FFFF])
{
let Some(c) = char::from_u32(cp) else {
continue;
};
match len {
1 => assert_eq!(utf8_decode(&[b0]), std_decode(&[b0])),
2 => {
for b1 in 0..=255u8 {
assert_eq!(
utf8_decode(&[b0, b1]),
std_decode(&[b0, b1]),
"{b0:x} {b1:x}"
);
}
}
3 => {
for b1 in 0..=255u8 {
for b2 in 0..=255u8 {
assert_eq!(
utf8_decode(&[b0, b1, b2]),
std_decode(&[b0, b1, b2]),
"{b0:x} {b1:x} {b2:x}"
);
}
}
}
_ => {
for b1 in 0..=255u8 {
for b2 in [0x00, 0x7F, 0x80, 0xA0, 0xBF, 0xC0] {
for b3 in [0x00, 0x7F, 0x80, 0xBF, 0xC0] {
assert_eq!(
utf8_decode(&[b0, b1, b2, b3]),
std_decode(&[b0, b1, b2, b3]),
"{b0:x} {b1:x} {b2:x} {b3:x}"
);
}
}
}
}
if c == '\x1b' {
continue;
}
let mut b = [0u8; 4];
let s = c.encode_utf8(&mut b);
assert_eq!(decode_utf8_replace(s.as_bytes(), &mut out), 1);
assert_eq!(out[0], cp);
}
}
// Regressions found by the differential fuzzer (utfcpp replace_invalid
// semantics: ONE U+FFFD per invalid sequence, trail bytes swallowed).
#[test]
fn decode_replace_swallows_trail_bytes() {
let mut out = [0u32; 16];
// overlong lead + trail
assert_eq!(decode_utf8_replace(b"\xc0\xa9", &mut out), 1);
assert_eq!(out[0], 0xFFFD);
// truncated 3-byte sequence followed by a non-trail byte
assert_eq!(decode_utf8_replace(b"\xe4\xa9\x41", &mut out), 2);
assert_eq!(&out[..2], &[0xFFFD, 0x41]);
// surrogate
assert_eq!(decode_utf8_replace(b"\xed\xa0\x80", &mut out), 1);
// invalid lead only skips itself
assert_eq!(decode_utf8_replace(b"\xff\x80", &mut out), 2);
// stray trail bytes are each invalid leads
assert_eq!(decode_utf8_replace(b"\x80\x80", &mut out), 2);
// truncated at end of input: one replacement
assert_eq!(decode_utf8_replace(b"\xf0\x9f", &mut out), 1);
}
#[test]
fn trim_partial_is_byte_value_based() {
assert_eq!(trim_partial_utf8(b"ab\xe4\x41"), 2);
assert_eq!(trim_partial_utf8(b"abc"), 3);
assert_eq!(trim_partial_utf8(b"ab\xff"), 2);
assert_eq!(trim_partial_utf8(b"\xe4\xb8"), 0);
assert_eq!(trim_partial_utf8(b""), 0);
}
#[test]
fn index_of_basic() {
assert_eq!(index_of(b"hello", b' '), None);
+36 -25
View File
@@ -347,29 +347,33 @@ impl<H: Handler> Stream<H> {
debug_assert!(input.len() <= cp_buf.len());
let mut offset = 0;
while offset < input.len() {
// If the scalar UTF-8 decoder was in the middle of processing a
// code sequence, we continue until it's not.
if self.utf8decoder.in_sequence() {
self.next_utf8(input[offset])?;
offset += 1;
continue;
}
// If we're not in the ground state then we process until we are.
// This can happen if the last chunk of input put us in the middle
// of a control sequence.
if self.parser.state != State::Ground {
offset += self.consume_until_ground(&input[offset..])?;
continue;
// If the scalar UTF-8 decoder was in the middle of processing a code
// sequence, we continue until it's not.
while self.utf8decoder.in_sequence() {
if offset >= input.len() {
return Ok(());
}
self.next_utf8(input[offset])?;
offset += 1;
}
if offset >= input.len() {
return Ok(());
}
if input[offset] == 0x1B {
offset += self.consume_all_escapes(&input[offset..])?;
continue;
}
// If we're not in the ground state then we process until we are. This
// can happen if the last chunk of input put us in the middle of a
// control sequence.
offset += self.consume_until_ground(&input[offset..])?;
if offset >= input.len() {
return Ok(());
}
offset += self.consume_all_escapes(&input[offset..])?;
// Ground state: everything up to the next ESC is UTF-8.
// If we're in the ground state then we can use SIMD to process input
// until we see an ESC (0x1B), since all other characters up to that
// point are just UTF-8.
while self.parser.state == State::Ground && offset < input.len() {
let res = simd::utf8_decode_until_control_seq(&input[offset..], cp_buf);
for &cp in &cp_buf[..res.decoded] {
if cp <= 0xF {
@@ -378,18 +382,25 @@ impl<H: Handler> Stream<H> {
self.print(cp)?;
}
}
// Consume the bytes we just processed.
offset += res.consumed;
if offset >= input.len() {
break;
return Ok(());
}
// If our offset is NOT an escape then we must have a partial or
// invalid UTF-8 sequence. Pass that byte to the scalar decoder,
// which handles replacement and carries partial state over.
// If our offset is NOT an escape then we must have a partial
// UTF-8 sequence. In that case, we pass the whole remainder off
// to the scalar parser.
if input[offset] != 0x1B {
self.next_utf8(input[offset])?;
offset += 1;
for &c in &input[offset..] {
self.next_utf8(c)?;
}
return Ok(());
}
// Process control sequences until we run out.
offset += self.consume_all_escapes(&input[offset..])?;
}
Ok(())
}