fix: preserve non-UTF-8 bytes in OS, config and CLI layers

Zig treats paths, arguments and environment values as byte slices; the
port converted them lossily in many places. Now (final-Z1.md):
- argv, env (incl. present non-UTF-8 values), HOME, PATH lookup,
  passwd, resources dir, TMPDIR, cgroup, config path canonicalization,
  +validate-config, +list-themes and Zig-string escapes (\xFF) keep raw
  bytes. String-based config values carry bytes via ghostty_os::osbytes,
  a bijective escape (noncharacter U+FDD0 + byte) that leaves all valid
  UTF-8 - including Nerd Font private-use icons - unchanged.
- os/file rlimit fallbacks exist on all targets.
- i18n locale canonicalization is an exact port of libintl's tables
  (e.g. pa-Guru -> pa_IN).
- New ghostty-build-info crate is the single source of version, channel,
  commit and build mode for all consumers.

Test Plan:
- cargo test -p ghostty-os -p ghostty-config -p ghostty-cli
  (incl. randomized osbytes round-trip property tests)

crates/ghostty-config/src/cli_args.rs

@@ -343,10 +343,21 @@ pub fn parse_auto_struct_with(
 
 /// Decode a double-quoted Zig string literal (including the quotes).
 /// Returns `None` on failure. Bytes produced by `\x` escapes that do not form
-/// valid UTF-8 are converted lossily.
+/// valid UTF-8 stay raw bytes: they are carried through the reversible
+/// `ghostty_os::osbytes` escape encoding (`\xFF` -> `U+FDD0 U+00FF`, decoded back
+/// to `0xFF` by `config_str_to_bytes`). See [`parse_zig_string_literal_bytes`]
+/// for the raw bytes.
 #[must_use]
 pub fn parse_zig_string_literal(s: &str) -> Option<String> {
-    let b = s.as_bytes();
+    parse_zig_string_literal_bytes(s).map(|b| ghostty_os::osbytes::bytes_to_config_string(&b))
+}
+
+/// Byte-exact [`parse_zig_string_literal`] (Zig `std.zig.string_literal`
+/// decoding yields a `[]u8`).
+#[must_use]
+pub fn parse_zig_string_literal_bytes(s: &str) -> Option<Vec<u8>> {
+    let b = ghostty_os::osbytes::config_str_to_bytes(s);
+    let b = b.as_slice();
     if b.len() < 2 || b[0] != b'"' || b[b.len() - 1] != b'"' {
         return None;
     }
@@ -395,7 +406,7 @@ pub fn parse_zig_string_literal(s: &str) -> Option<String> {
         }
         i += 1;
     }
-    Some(String::from_utf8_lossy(&out).into_owned())
+    Some(out)
 }
 
 /// Parse a "packed struct" value (comma separated names, optionally
@@ -652,8 +663,13 @@ impl<I: Iterator<Item = String>> ArgIter for ArgsIterator<I> {
 /// Create an args iterator for the process args. This will skip argv0.
 #[must_use]
 pub fn args_iterator() -> ArgsIterator<std::iter::Skip<std::vec::IntoIter<String>>> {
-    // Byte-safe: never panics on non-UTF-8 arguments.
-    ArgsIterator::new(ghostty_os::args::collect_lossy().into_iter().skip(1))
+    // Byte-safe: non-UTF-8 arguments keep their bytes (reversible escape
+    // encoding, see `ghostty_os::osbytes`).
+    ArgsIterator::new(
+        ghostty_os::args::collect_config_strings()
+            .into_iter()
+            .skip(1),
+    )
 }
 
 /// An iterator valid for arg parsing from a slice.
@@ -791,7 +807,8 @@ impl<R: BufRead> LineIterator<R> {
 
             self.line += 1;
 
-            let line = String::from_utf8_lossy(&buf);
+            // Raw bytes survive via the reversible escape encoding.
+            let line = ghostty_os::osbytes::bytes_to_config_string(&buf);
             // Trim any whitespace (including CR) around it
             let trim = line.trim_matches(|c| c == ' ' || c == '\t' || c == '\r');
 
@@ -1533,4 +1550,42 @@ mod tests {
         );
         assert_eq!(parse_zig_string_literal("\"\\q\""), None);
     }
+
+    #[test]
+    fn zig_string_literal_keeps_raw_bytes() {
+        // `\xFF` is not valid UTF-8 and must stay the single byte 0xFF.
+        assert_eq!(
+            parse_zig_string_literal_bytes("\"a\\xFFb\"").unwrap(),
+            vec![b'a', 0xff, b'b']
+        );
+        let s = parse_zig_string_literal("\"a\\xFFb\"").unwrap();
+        assert_eq!(
+            ghostty_os::osbytes::config_str_to_bytes(&s),
+            vec![b'a', 0xff, b'b']
+        );
+        assert!(!s.contains('\u{fffd}'));
+        // Valid multi-byte sequences spelled with escapes still decode.
+        assert_eq!(
+            parse_zig_string_literal("\"\\xC3\\xA9\"").unwrap(),
+            "\u{e9}"
+        );
+        // Raw (unescaped) invalid input bytes survive the escape encoding.
+        let raw = ghostty_os::osbytes::bytes_to_config_string(b"\"x\xfey\"");
+        assert_eq!(
+            parse_zig_string_literal_bytes(&raw).unwrap(),
+            vec![b'x', 0xfe, b'y']
+        );
+    }
+
+    #[test]
+    fn line_iterator_keeps_invalid_utf8_bytes() {
+        let data: &[u8] = b"title = a\xffb\n";
+        let mut it = LineIterator::new(data);
+        let line = it.next().unwrap();
+        assert_eq!(
+            ghostty_os::osbytes::config_str_to_bytes(&line),
+            b"--title=a\xffb".to_vec()
+        );
+        assert!(!line.contains('\u{fffd}'));
+    }
 }

crates/ghostty-config/src/config.rs

@@ -3877,7 +3877,7 @@ impl Config {
     ///
     /// `path` must be resolved and absolute.
     pub fn load_file(&mut self, path: &str) -> Result<(), Error> {
-        assert!(std::path::Path::new(path).is_absolute());
+        assert!(ghostty_os::osbytes::config_str_to_path(path).is_absolute());
         let file = match crate::file_load::open(path) {
             Ok(f) => f,
             Err(crate::file_load::OpenFileError::NotAFile) => {
@@ -3916,9 +3916,9 @@ impl Config {
         }
         let mut iter = cli_args::LineIterator::with_path(reader, path);
         self.load_iter(&mut iter)?;
-        let dir = std::path::Path::new(path)
+        let dir = ghostty_os::osbytes::config_str_to_path(path)
             .parent()
-            .map(|p| p.to_string_lossy().into_owned())
+            .map(ghostty_os::osbytes::path_to_config_string)
             .unwrap_or_default();
         self.expand_paths(&dir)?;
         Ok(())
@@ -4067,7 +4067,7 @@ impl Config {
                 // a command to execute.
                 let mut builder: Vec<String> = Vec::new();
                 for arg in args {
-                    let copy = arg.to_string_lossy().into_owned();
+                    let copy = ghostty_os::osbytes::bytes_to_config_string(arg.to_bytes());
                     self.replay_steps.push(ReplayStep::Arg(copy.clone()));
                     builder.push(copy);
                 }
@@ -4126,7 +4126,7 @@ impl Config {
         // Any paths referenced from the CLI are relative to the current working
         // directory.
         let cwd = std::fs::canonicalize(".")?;
-        self.expand_paths(&cwd.to_string_lossy())?;
+        self.expand_paths(&ghostty_os::osbytes::path_to_config_string(&cwd))?;
         Ok(())
     }
 
@@ -4170,7 +4170,7 @@ impl Config {
 
             // All paths should already be absolute at this point because
             // they're fixed up after each load.
-            assert!(std::path::Path::new(&path).is_absolute());
+            assert!(ghostty_os::osbytes::config_str_to_path(&path).is_absolute());
 
             // We must only load a unique file once
             if !loaded.insert(path.clone()) {
@@ -4183,7 +4183,7 @@ impl Config {
                 continue;
             }
 
-            let file = match std::fs::File::open(&path) {
+            let file = match std::fs::File::open(ghostty_os::osbytes::config_str_to_path(&path)) {
                 Ok(f) => f,
                 Err(err) => {
                     if err.kind() != std::io::ErrorKind::NotFound || !optional {

crates/ghostty-config/src/file_load.rs

@@ -59,7 +59,7 @@ pub fn legacy_default_app_support_path() -> Result<String, xdg::Error> {
 #[cfg(target_os = "macos")]
 fn app_support_dir(sub_path: &str) -> Result<String, xdg::Error> {
     ghostty_os::macos::app_support_dir(sub_path)
-        .map(|p| p.to_string_lossy().into_owned())
+        .map(|p| ghostty_os::osbytes::path_to_config_string(&p))
         .map_err(|_| xdg::Error::AppleApiFailed)
 }
 
@@ -159,9 +159,10 @@ impl std::error::Error for OpenFileError {}
 /// if it exists and is non-empty. This also constrains the possible
 /// errors to a smaller set that we can explicitly handle.
 pub fn open(path: &str) -> Result<File, OpenFileError> {
-    assert!(std::path::Path::new(path).is_absolute());
+    assert!(ghostty_os::osbytes::config_str_to_path(path).is_absolute());
 
-    let file = File::open(path).map_err(|err| {
+    // `path` may carry the reversible byte escape of non-UTF-8 names.
+    let file = File::open(ghostty_os::osbytes::config_str_to_os(path)).map_err(|err| {
         if err.kind() == std::io::ErrorKind::NotFound {
             OpenFileError::FileNotFound
         } else {

crates/ghostty-config/src/path.rs

@@ -2,9 +2,11 @@
 
 use std::{
     fmt,
-    path::{Component, Path as StdPath, PathBuf},
+    path::{Component, PathBuf},
 };
 
+use ghostty_os::osbytes::{config_str_to_path, path_to_config_string};
+
 use crate::{
     c_value::CValue,
     cli_args::{ArgError, ConfigField},
@@ -32,7 +34,7 @@ impl Default for Path {
 
 /// Lexically resolve `base/path` like `std.fs.path.resolve`.
 fn resolve_lexical(base: &str, path: &str) -> String {
-    let joined = PathBuf::from(base).join(path);
+    let joined = config_str_to_path(base).join(config_str_to_path(path));
     let mut out = PathBuf::new();
     for c in joined.components() {
         match c {
@@ -43,7 +45,7 @@ fn resolve_lexical(base: &str, path: &str) -> String {
             other => out.push(other.as_os_str()),
         }
     }
-    out.to_string_lossy().into_owned()
+    path_to_config_string(&out)
 }
 
 impl Path {
@@ -109,12 +111,15 @@ impl Path {
     /// absolute path. Errors are added to `diags` if they occur; an `Err` is
     /// only returned if the base directory cannot be opened.
     pub fn expand(&mut self, base: &str, diags: &mut DiagnosticList) -> std::io::Result<()> {
-        assert!(StdPath::new(base).is_absolute());
+        // Config strings carry non-UTF-8 bytes via a reversible escape
+        // encoding; filesystem calls use the decoded OS paths.
+        let base_path = config_str_to_path(base);
+        assert!(base_path.is_absolute());
 
         let path = self.path().to_string();
 
         // If it is already absolute we can ignore it.
-        if path.is_empty() || StdPath::new(&path).is_absolute() {
+        if path.is_empty() || config_str_to_path(&path).is_absolute() {
             return Ok(());
         }
 
@@ -143,11 +148,11 @@ impl Path {
         }
 
         // Make sure the base directory can be opened.
-        std::fs::read_dir(base)?;
+        std::fs::read_dir(&base_path)?;
 
-        let full = StdPath::new(base).join(&path);
+        let full = base_path.join(config_str_to_path(&path));
         let abs = match std::fs::canonicalize(&full) {
-            Ok(p) => p.to_string_lossy().into_owned(),
+            Ok(p) => path_to_config_string(&p),
             Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
                 // The file doesn't exist. Try to resolve the relative path
                 // another way.
@@ -374,4 +379,23 @@ mod tests {
         assert_eq!(p, Path::Optional("/abs/path".into()));
         assert!(diags.is_empty());
     }
+
+    #[cfg(unix)]
+    #[test]
+    fn expand_symlink_to_non_utf8_target_round_trips() {
+        use std::os::unix::ffi::OsStrExt;
+        let tmp = ghostty_os::TempDir::init().unwrap();
+        let base = std::fs::canonicalize(tmp.path()).unwrap();
+        let target = base.join(std::ffi::OsStr::from_bytes(b"t\xffarget"));
+        std::fs::write(&target, b"x").unwrap();
+        std::os::unix::fs::symlink(&target, base.join("link")).unwrap();
+
+        let mut p = Path::Required("link".into());
+        let mut diags = DiagnosticList::default();
+        p.expand(&path_to_config_string(&base), &mut diags).unwrap();
+        assert_eq!(diags.items().len(), 0);
+        // The canonical (non-UTF-8) target is preserved byte for byte.
+        assert_eq!(config_str_to_path(p.path()), target);
+        assert!(std::fs::metadata(config_str_to_path(p.path())).is_ok());
+    }
 }

crates/ghostty-config/src/theme.rs

@@ -3,7 +3,14 @@
 //! Zig reads the resources dir from global state; here it is passed
 //! explicitly (`resources_dir`, the value of `ResourcesDir.app()`).
 
-use std::{fs::File, path::Path};
+use std::fs::File;
+
+use ghostty_os::osbytes::{
+    config_str_to_os,
+    config_str_to_path,
+    os_to_config_string,
+    path_to_config_string,
+};
 
 use crate::diagnostics::{Diagnostic, DiagnosticList};
 
@@ -35,12 +42,9 @@ impl Location {
             .ok(),
             Location::Resources => {
                 let base = resources_dir?;
-                Some(
-                    Path::new(base)
-                        .join("themes")
-                        .to_string_lossy()
-                        .into_owned(),
-                )
+                Some(path_to_config_string(
+                    &config_str_to_path(base).join("themes"),
+                ))
             }
         }
     }
@@ -161,7 +165,7 @@ pub fn open(
     diags: &mut DiagnosticList,
 ) -> Option<OpenedTheme> {
     // Absolute themes are loaded a different path.
-    if Path::new(theme).is_absolute() {
+    if config_str_to_path(theme).is_absolute() {
         let file = open_absolute(theme, diags)?;
         if !check_regular(&file, theme, diags) {
             return None;
@@ -172,9 +176,9 @@ pub fn open(
         });
     }
 
-    let basename = Path::new(theme)
+    let basename = config_str_to_path(theme)
         .file_name()
-        .map(|s| s.to_string_lossy().into_owned())
+        .map(os_to_config_string)
         .unwrap_or_default();
     if theme != basename {
         diag(
@@ -190,8 +194,8 @@ pub fn open(
     // one that exists.
     let mut it = LocationIterator::new(resources_dir);
     while let Some((_, dir)) = it.next_location() {
-        let path = Path::new(&dir).join(theme).to_string_lossy().into_owned();
-        match File::open(&path) {
+        let path = path_to_config_string(&config_str_to_path(&dir).join(config_str_to_os(theme)));
+        match File::open(config_str_to_path(&path)) {
             Ok(file) => {
                 if !check_regular(&file, theme, diags) {
                     return None;
@@ -215,7 +219,7 @@ pub fn open(
     // our iterator, reiterate over in order to build a better error message.
     it.reset();
     while let Some((_, dir)) = it.next_location() {
-        let path = Path::new(&dir).join(theme).to_string_lossy().into_owned();
+        let path = path_to_config_string(&config_str_to_path(&dir).join(config_str_to_os(theme)));
         diag(
             diags,
             format!("theme \"{theme}\" not found, tried path \"{path}\""),
@@ -229,7 +233,7 @@ pub fn open(
 /// then messages will be appended to the given error list and None is
 /// returned.
 pub fn open_absolute(theme: &str, diags: &mut DiagnosticList) -> Option<File> {
-    match File::open(theme) {
+    match File::open(config_str_to_path(theme)) {
         Ok(f) => Some(f),
         Err(err) => {
             if err.kind() == std::io::ErrorKind::NotFound {