perf: add ghostty-bench and bring terminal throughput to Zig parity
Adds crates/ghostty-bench, a port of ZIG/src/benchmark and src/synthetic: the ghostty-bench binary (terminal-stream, terminal-parser, codepoint-width, grapheme-break, is-symbol, osc-parser, screen-clone with Zig's CLI) and ghostty-synthetic data generators (+ascii, +utf8, +osc, plus a Rust-only +sgr mode). Measured against Ghostty's own ghostty-bench built with Zig 0.15.2 (ReleaseFast) vs --profile production, the hot paths were optimized without behaviour changes: - Terminal::print single-lookup fast path for width-1 chars mid-line - page grapheme allocator: bitmap first-fit instead of a linear scan (was 38% of utf8 stream time; randomized equivalence test added) - plain cells skip per-cell work in clear_cells and row cloning - ASCII fast path + strict hand-written UTF-8 decoder (exhaustive equivalence test against std) - parser/OSC parser split into inlined fast and state-change paths - RenderState allocates grapheme vectors only for grapheme rows - Page::assert_integrity no-op stub so the production profile builds terminal-stream is now at parity (ascii 1.08x, utf8 0.97x, SGR 1.04x of Zig's time). Still slower: screen-clone (~2.6x, zero-filling fresh cell Vecs vs Zig's lazily-zeroed mmap), osc-parser (1.9x), and the raw parser on SGR data (1.5x). Full table in docs/rewrite/notes/perf-P1.md. Test Plan: - cargo test --workspace (no failures) - cargo run --profile production -p ghostty-bench -- +terminal-stream --data=<file> and compare with Zig's ghostty-bench
This commit is contained in:
ddidderr committed
1 parent
c6a0c49567
commit
062c643ea8
18 files changed
+2195
-82
No files matched your search
@@ -515,6 +515,27 @@ impl From<AddError> for Error {
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Hasher for cell-index keys: indices are already well distributed, so a
|
||||
/// single multiply replaces SipHash on the print path.
|
||||
#[derive(Debug, Default, Clone, Copy)]
|
||||
struct IndexHasher(u64);
|
||||
impl std::hash::Hasher for IndexHasher {
|
||||
#[inline]
|
||||
fn finish(&self) -> u64 {
|
||||
self.0
|
||||
}
|
||||
#[inline]
|
||||
fn write(&mut self, bytes: &[u8]) {
|
||||
for &b in bytes {
|
||||
self.0 = (self.0.rotate_left(5) ^ u64::from(b)).wrapping_mul(0x517c_c1b7_2722_0a95);
|
||||
}
|
||||
}
|
||||
#[inline]
|
||||
fn write_usize(&mut self, n: usize) {
|
||||
self.0 = (n as u64).wrapping_mul(0x9e37_79b9_7f4a_7c15);
|
||||
}
|
||||
}
|
||||
type IndexMap<V> = HashMap<usize, V, std::hash::BuildHasherDefault<IndexHasher>>;
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct Allocation {
|
||||
start: usize,
|
||||
@@ -522,43 +543,91 @@ struct Allocation {
|
||||
}
|
||||
#[derive(Debug, Clone)]
|
||||
struct ChunkAlloc {
|
||||
used: Vec<bool>,
|
||||
/// Bitmap of used chunks (bit set = used), 64 chunks per word.
|
||||
used: Vec<u64>,
|
||||
/// Total number of chunks.
|
||||
chunks: usize,
|
||||
/// Every word below this index is completely used.
|
||||
first_free_word: usize,
|
||||
chunk: usize,
|
||||
}
|
||||
impl ChunkAlloc {
|
||||
fn new(layout: AllocLayout, chunk: usize) -> Result<Self, Error> {
|
||||
let mut used = Vec::new();
|
||||
let count = layout.bitmap_count * 64;
|
||||
used.try_reserve_exact(count)
|
||||
used.try_reserve_exact(layout.bitmap_count)
|
||||
.map_err(|_| Error::OutOfMemory)?;
|
||||
used.resize(count, false);
|
||||
Ok(Self { used, chunk })
|
||||
used.resize(layout.bitmap_count, 0);
|
||||
Ok(Self {
|
||||
used,
|
||||
chunks: layout.bitmap_count * 64,
|
||||
first_free_word: 0,
|
||||
chunk,
|
||||
})
|
||||
}
|
||||
/// First-fit allocation of `bytes` (rounded up to whole chunks): the
|
||||
/// lowest-addressed run of free chunks is used.
|
||||
fn alloc(&mut self, bytes: usize) -> Option<Allocation> {
|
||||
let n = bytes.div_ceil(self.chunk);
|
||||
if n == 0 {
|
||||
return Some(Allocation { start: 0, len: 0 });
|
||||
}
|
||||
let mut run = 0;
|
||||
for i in 0..self.used.len() {
|
||||
if self.used[i] {
|
||||
while self.first_free_word < self.used.len() && self.used[self.first_free_word] == u64::MAX
|
||||
{
|
||||
self.first_free_word += 1;
|
||||
}
|
||||
let mut run = 0usize;
|
||||
let mut run_start = 0usize;
|
||||
for w in self.first_free_word..self.used.len() {
|
||||
let word = self.used[w];
|
||||
if word == u64::MAX {
|
||||
run = 0;
|
||||
} else {
|
||||
run += 1;
|
||||
continue;
|
||||
}
|
||||
if run == n {
|
||||
let start = i + 1 - n;
|
||||
self.used[start..=i].fill(true);
|
||||
return Some(Allocation { start, len: n });
|
||||
if word == 0 {
|
||||
if run == 0 {
|
||||
run_start = w * 64;
|
||||
}
|
||||
run += 64;
|
||||
if run >= n {
|
||||
return Some(self.take(run_start, n));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if n == 1 {
|
||||
let b = word.trailing_ones() as usize;
|
||||
return Some(self.take(w * 64 + b, 1));
|
||||
}
|
||||
for b in 0..64 {
|
||||
if word >> b & 1 == 1 {
|
||||
run = 0;
|
||||
} else {
|
||||
if run == 0 {
|
||||
run_start = w * 64 + b;
|
||||
}
|
||||
run += 1;
|
||||
if run == n {
|
||||
return Some(self.take(run_start, n));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
fn take(&mut self, start: usize, len: usize) -> Allocation {
|
||||
for i in start..start + len {
|
||||
self.used[i / 64] |= 1 << (i % 64);
|
||||
}
|
||||
Allocation { start, len }
|
||||
}
|
||||
fn free(&mut self, a: Allocation) {
|
||||
self.used[a.start..a.start + a.len].fill(false);
|
||||
for i in a.start..a.start + a.len {
|
||||
self.used[i / 64] &= !(1 << (i % 64));
|
||||
}
|
||||
self.first_free_word = self.first_free_word.min(a.start / 64);
|
||||
}
|
||||
fn reset(&mut self) {
|
||||
self.used.fill(false);
|
||||
self.used.fill(0);
|
||||
self.first_free_word = 0;
|
||||
}
|
||||
}
|
||||
/// Page-owned rows, cells, and bounded auxiliary storage.
|
||||
@@ -579,9 +648,9 @@ pub struct Page {
|
||||
/// Shared hyperlink values.
|
||||
pub hyperlink_set: hyperlink::Set,
|
||||
// Graphemes share a page arena; copying a cell never allocates a Vec.
|
||||
grapheme_map: HashMap<usize, std::ops::Range<usize>>,
|
||||
grapheme_map: IndexMap<std::ops::Range<usize>>,
|
||||
grapheme_data: Vec<u32>,
|
||||
grapheme_allocations: HashMap<usize, Allocation>,
|
||||
grapheme_allocations: IndexMap<Allocation>,
|
||||
grapheme_alloc: ChunkAlloc,
|
||||
hyperlink_map: hyperlink::Map,
|
||||
string_alloc: ChunkAlloc,
|
||||
@@ -683,16 +752,16 @@ impl Page {
|
||||
cells.resize(count, Cell::default());
|
||||
let grapheme_alloc = ChunkAlloc::new(l.grapheme_alloc_layout, 16)?;
|
||||
let mut grapheme_data = Vec::new();
|
||||
let codepoints = grapheme_alloc.used.len() * 4;
|
||||
let codepoints = grapheme_alloc.chunks * 4;
|
||||
grapheme_data
|
||||
.try_reserve_exact(codepoints)
|
||||
.map_err(|_| Error::OutOfMemory)?;
|
||||
grapheme_data.resize(codepoints, 0);
|
||||
let mut grapheme_map = HashMap::new();
|
||||
let mut grapheme_map = IndexMap::default();
|
||||
grapheme_map
|
||||
.try_reserve(l.grapheme_map_layout.capacity)
|
||||
.map_err(|_| Error::OutOfMemory)?;
|
||||
let mut grapheme_allocations = HashMap::new();
|
||||
let mut grapheme_allocations = IndexMap::default();
|
||||
grapheme_allocations
|
||||
.try_reserve(l.grapheme_map_layout.capacity)
|
||||
.map_err(|_| Error::OutOfMemory)?;
|
||||
@@ -1074,7 +1143,9 @@ impl Page {
|
||||
}
|
||||
let old = self.rows[y];
|
||||
let full = start == 0 && end == usize::from(self.size.cols) && !protect;
|
||||
// Only needed when the row stays (possibly) styled, i.e. a partial clear.
|
||||
let styled = old.styled()
|
||||
&& !full
|
||||
&& self
|
||||
.get_cells(y)
|
||||
.is_some_and(|cells| cells.iter().any(|c| c.has_styling()));
|
||||
@@ -1087,6 +1158,15 @@ impl Page {
|
||||
self.rows[y].set_kitty_virtual_placeholder(false);
|
||||
}
|
||||
let base = self.rows[y].cells();
|
||||
|
||||
// Fast path: the row flags are a superset of what its cells carry, so
|
||||
// with none of them set there is nothing to release per cell.
|
||||
if !protect && !old.styled() && !old.grapheme() && !old.hyperlink() {
|
||||
self.cells[base + start..base + end].fill(Cell::default());
|
||||
self.rows[y].set_dirty(true);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
for i in base + start..base + end {
|
||||
let c = self.cells[i];
|
||||
if protect && c.protected() {
|
||||
@@ -1104,10 +1184,18 @@ impl Page {
|
||||
self.cells[i] = Cell::default();
|
||||
}
|
||||
if old.grapheme() {
|
||||
self.update_row_grapheme_flag(y);
|
||||
if full {
|
||||
self.rows[y].set_grapheme(false);
|
||||
} else {
|
||||
self.update_row_grapheme_flag(y);
|
||||
}
|
||||
}
|
||||
if old.hyperlink() {
|
||||
self.update_row_hyperlink_flag(y);
|
||||
if full {
|
||||
self.rows[y].set_hyperlink(false);
|
||||
} else {
|
||||
self.update_row_hyperlink_flag(y);
|
||||
}
|
||||
}
|
||||
if old.styled() {
|
||||
self.rows[y].set_styled(!full && styled);
|
||||
@@ -1244,10 +1332,26 @@ impl Page {
|
||||
i: usize,
|
||||
x: usize,
|
||||
y: usize,
|
||||
) -> Result<(), Error> {
|
||||
self.clear_cells(y, x, x + 1, false)?;
|
||||
self.clone_cell_from_cleared(src, i, x, y)
|
||||
}
|
||||
/// Like [`Page::clone_cell_from`] but the destination cell must already
|
||||
/// have been cleared (e.g. its whole row range was cleared up front).
|
||||
fn clone_cell_from_cleared(
|
||||
&mut self,
|
||||
src: &Self,
|
||||
i: usize,
|
||||
x: usize,
|
||||
y: usize,
|
||||
) -> Result<(), Error> {
|
||||
let cell = src.cells[i];
|
||||
let dst = self.cell_index(x, y)?;
|
||||
self.clear_cells(y, x, x + 1, false)?;
|
||||
// Plain cells (the vast majority) carry no managed memory.
|
||||
if !cell.has_grapheme() && !cell.hyperlink() && !cell.has_styling() {
|
||||
self.cells[dst] = cell;
|
||||
return Ok(());
|
||||
}
|
||||
self.cells[dst] = cell;
|
||||
self.cells[dst].set_style_id(0);
|
||||
self.cells[dst].set_hyperlink(false);
|
||||
@@ -1379,9 +1483,17 @@ impl Page {
|
||||
let original_row = *self.get_row(dst_y).ok_or(Error::InvalidCoordinate)?;
|
||||
self.clear_cells(dst_y, dst_x, dst_x + len, false)?;
|
||||
if source_row.managed_memory() {
|
||||
// Row metadata/cell indices were validated above (`get_row`).
|
||||
let from = source_row.cells() + src_x;
|
||||
let to = original_row.cells() + dst_x;
|
||||
for n in 0..len {
|
||||
let i = src.cell_index(src_x + n, src_y)?;
|
||||
self.clone_cell_from(src, i, dst_x + n, dst_y)?;
|
||||
let cell = src.cells[from + n];
|
||||
if !cell.has_grapheme() && !cell.hyperlink() && !cell.has_styling() {
|
||||
// Plain cell: no managed memory to copy.
|
||||
self.cells[to + n] = cell;
|
||||
} else {
|
||||
self.clone_cell_from_cleared(src, from + n, dst_x + n, dst_y)?;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let from = source_row.cells() + src_x;
|
||||
@@ -1640,7 +1752,14 @@ impl Page {
|
||||
}
|
||||
}
|
||||
|
||||
fn swap_map<T>(map: &mut HashMap<usize, T>, a: usize, b: usize) {
|
||||
#[cfg(not(any(debug_assertions, test)))]
|
||||
impl Page {
|
||||
/// No-op in release builds (the integrity checks are debug-only).
|
||||
#[inline(always)]
|
||||
pub fn assert_integrity(&self) {}
|
||||
}
|
||||
|
||||
fn swap_map<T, S: std::hash::BuildHasher>(map: &mut HashMap<usize, T, S>, a: usize, b: usize) {
|
||||
let av = map.remove(&a);
|
||||
let bv = map.remove(&b);
|
||||
if let Some(v) = av {
|
||||
@@ -1653,6 +1772,53 @@ fn swap_map<T>(map: &mut HashMap<usize, T>, a: usize, b: usize) {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn chunk_alloc_matches_naive_first_fit() {
|
||||
let mut fast = ChunkAlloc::new(
|
||||
AllocLayout {
|
||||
bitmap_count: 5,
|
||||
..AllocLayout::default()
|
||||
},
|
||||
16,
|
||||
)
|
||||
.unwrap();
|
||||
let mut naive = vec![false; 5 * 64];
|
||||
let mut live: Vec<Allocation> = Vec::new();
|
||||
let mut seed = 0x1234_5678_9abc_def0u64;
|
||||
let mut rnd = move || {
|
||||
seed ^= seed << 13;
|
||||
seed ^= seed >> 7;
|
||||
seed ^= seed << 17;
|
||||
seed
|
||||
};
|
||||
for _ in 0..20_000 {
|
||||
if rnd() % 3 != 0 || live.is_empty() {
|
||||
let n = (rnd() % 70 + 1) as usize;
|
||||
let expect = (|| {
|
||||
let mut run = 0;
|
||||
for (i, &used) in naive.iter().enumerate() {
|
||||
run = if used { 0 } else { run + 1 };
|
||||
if run == n {
|
||||
return Some(i + 1 - n);
|
||||
}
|
||||
}
|
||||
None
|
||||
})();
|
||||
let got = fast.alloc(n * 16);
|
||||
assert_eq!(got.map(|a| a.start), expect);
|
||||
if let Some(a) = got {
|
||||
naive[a.start..a.start + a.len].fill(true);
|
||||
live.push(a);
|
||||
}
|
||||
} else {
|
||||
let i = (rnd() as usize) % live.len();
|
||||
let a = live.swap_remove(i);
|
||||
naive[a.start..a.start + a.len].fill(false);
|
||||
fast.free(a);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
use crate::style::{Color, Flags};
|
||||
fn page() -> Page {
|
||||
Page::init(Capacity {
|
||||
|
||||
@@ -547,6 +547,19 @@ impl PageList {
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
/// Mutable access to a tracked pin's location for hot paths that only
|
||||
/// change `x` within the same row (cursor movement while printing).
|
||||
/// Unlike [`PageList::set_tracked_pin`] this performs no node validation
|
||||
/// and does not invalidate the cached viewport offset, so it must not be
|
||||
/// used to move the pin to another row or node.
|
||||
#[inline]
|
||||
pub fn tracked_pin_x_mut(&mut self, h: TrackedPin) -> Option<&mut Pin> {
|
||||
self.pins
|
||||
.get_mut(h.index)
|
||||
.filter(|s| s.generation == h.generation)?
|
||||
.pin
|
||||
.as_mut()
|
||||
}
|
||||
/// Release a tracked handle. The internal viewport handle remains owned.
|
||||
pub fn untrack_pin(&mut self, h: TrackedPin) {
|
||||
if self.viewport_pin == Some(h) {
|
||||
|
||||
@@ -549,7 +549,6 @@ impl RenderState {
|
||||
continue;
|
||||
}
|
||||
|
||||
row.cells.grapheme.resize_with(cols, Vec::new);
|
||||
row.cells.style.resize(cols, Style::default());
|
||||
for (x, page_cell) in page_cells.iter().enumerate() {
|
||||
// Append assuming its a single-codepoint, styled cell
|
||||
@@ -569,6 +568,11 @@ impl RenderState {
|
||||
if let Ok(idx) = p.cell_index(x, usize::from(row_pin.y))
|
||||
&& let Some(g) = p.lookup_grapheme(idx)
|
||||
{
|
||||
// Only rows that actually contain graphemes pay
|
||||
// for the per-cell vectors.
|
||||
if row.cells.grapheme.is_empty() {
|
||||
row.cells.grapheme.resize_with(cols, Vec::new);
|
||||
}
|
||||
row.cells.grapheme[x].extend_from_slice(g);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -613,6 +613,12 @@ impl Terminal {
|
||||
// We have a printable character, save it
|
||||
self.previous_char = Some(c);
|
||||
|
||||
// Fast path: the overwhelmingly common case of printing a single-width
|
||||
// character in the middle of a line onto a plain narrow cell.
|
||||
if width == 1 && !self.modes.get(Mode::Insert) && self.print_fast(c, right_limit) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// If we're soft-wrapping, then handle that first.
|
||||
if self.screen().cursor.pending_wrap && self.modes.get(Mode::Wraparound) {
|
||||
self.print_wrap()?;
|
||||
@@ -693,6 +699,66 @@ impl Terminal {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Fast path of `print` for a single-width character that does not hit
|
||||
/// the cursor's right limit. Performs exactly what the general path does
|
||||
/// (mark dirty, write the cell, move right) with a single page lookup.
|
||||
/// Returns `false` without touching anything if any precondition for the
|
||||
/// shortcut does not hold; the caller then takes the general path.
|
||||
#[inline]
|
||||
fn print_fast(&mut self, c: u32, right_limit: size::CellCountInt) -> bool {
|
||||
let scr = self.screens.active_mut();
|
||||
if scr.cursor.pending_wrap
|
||||
|| scr.cursor.hyperlink_id != 0
|
||||
|| scr.charset.single_shift.is_some()
|
||||
|| c == KITTY_PLACEHOLDER
|
||||
|| usize::from(scr.cursor.x) + 1 >= usize::from(right_limit)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let set = scr.charset.charsets.get(scr.charset.gl);
|
||||
if set != Charset::Utf8 && set != Charset::Ascii {
|
||||
return false;
|
||||
}
|
||||
|
||||
let style_id = scr.cursor.style_id;
|
||||
let protected = scr.cursor.protected;
|
||||
let semantic = scr.cursor.semantic_content;
|
||||
let pin_handle = scr.cursor.page_pin;
|
||||
let Some(pin) = scr.pages.tracked_pin(pin_handle) else {
|
||||
return false;
|
||||
};
|
||||
let (x, y) = (usize::from(pin.x), usize::from(pin.y));
|
||||
let page = scr.page_mut(pin.node);
|
||||
let Ok(idx) = page.cell_index(x, y) else {
|
||||
return false;
|
||||
};
|
||||
let old = page.cells[idx];
|
||||
if old.wide() != Wide::Narrow
|
||||
|| old.has_grapheme()
|
||||
|| old.hyperlink()
|
||||
|| old.style_id() != style_id
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Same cell value as `print_cell` produces.
|
||||
let mut new_cell = Cell::init(c);
|
||||
new_cell.set_content_tag(ContentTag::Codepoint);
|
||||
new_cell.set_style_id(style_id);
|
||||
new_cell.set_wide(Wide::Narrow);
|
||||
new_cell.set_protected(protected);
|
||||
new_cell.set_semantic_content(semantic);
|
||||
page.cells[idx] = new_cell;
|
||||
page.rows[y].set_dirty(true);
|
||||
|
||||
// Move the cursor right (`Screen::cursor_right(1)`).
|
||||
if let Some(p) = scr.pages.tracked_pin_x_mut(pin_handle) {
|
||||
p.x += 1;
|
||||
}
|
||||
scr.cursor.x += 1;
|
||||
true
|
||||
}
|
||||
|
||||
/// The grapheme-clustering section of `print`. Returns `true` if the
|
||||
/// character was fully handled (Zig: `return`), `false` if printing should
|
||||
/// continue on the normal path (Zig: `break :grapheme`).
|
||||
|
||||
Reference in new issue
Block a user