//! Three-level lookup tables for Unicode properties. //! //! This whole file is based on the algorithm described here: //! //! //! NOTE: this file is also `#[path]`-included by `build.rs`, so it must not //! depend on anything else in the crate. #![allow(dead_code)] #![allow( clippy::unwrap_used, clippy::single_match_else, clippy::checked_conversions, clippy::trivially_copy_pass_by_ref, clippy::expect_used )] use std::{collections::HashMap, fmt::Write as _}; const BLOCK_SIZE: usize = 256; /// Number of codepoints covered by the tables (`maxInt(u21) + 1`). pub const CODEPOINT_COUNT: usize = 0x20_0000; /// Errors from table generation. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum GenerateError { BlockTooLarge, Stage2TooLarge, } /// Owned lookup tables, the output of [`generate`]. #[derive(Debug, Clone)] pub struct OwnedTables { pub stage1: Vec, pub stage2: Vec, pub stage3: Vec, } /// Generate the three-level lookup table for all `u21` codepoints. /// `get` returns the mapping for a codepoint; mappings are compared with `==` /// (the Zig `Context.eql`). pub fn generate( mut get: impl FnMut(u32) -> E, ) -> Result, GenerateError> { let mut blocks_map: HashMap, u16> = HashMap::new(); let mut stage1: Vec = Vec::new(); let mut stage2: Vec = Vec::new(); let mut stage3: Vec = Vec::new(); let mut block: Vec = Vec::with_capacity(BLOCK_SIZE); for cp in 0..CODEPOINT_COUNT { // Get our block value and find the matching result value in our list // of possible values in stage3. This way, each possible mapping only // gets one entry in stage3. let elem = get(cp as u32); let idx = match stage3.iter().position(|item| *item == elem) { Some(i) => i, None => { stage3.push(elem); stage3.len() - 1 } }; block.push(u16::try_from(idx).map_err(|_| GenerateError::BlockTooLarge)?); if block.len() < BLOCK_SIZE && cp != CODEPOINT_COUNT - 1 { continue; } block.resize(BLOCK_SIZE, 0); let next = stage2.len(); let s2 = match blocks_map.get(&block) { Some(&s) => s, None => { let s = u16::try_from(next).map_err(|_| GenerateError::Stage2TooLarge)?; stage2.extend_from_slice(&block); blocks_map.insert(block.clone(), s); s } }; stage1.push(s2); block.clear(); } assert!(stage1.len() <= usize::from(u16::MAX)); assert!(stage2.len() <= usize::from(u16::MAX)); assert!(stage3.len() <= usize::from(u16::MAX)); Ok(OwnedTables { stage1, stage2, stage3, }) } impl OwnedTables { /// Borrow as lookup [`Tables`]. #[must_use] pub fn as_tables(&self) -> Tables<'_, E> { Tables { stage1: &self.stage1, stage2: &self.stage2, stage3: &self.stage3, } } /// Writes the lookup table as Rust source. The output defines three /// statics (`STAGE1`, `STAGE2`, `STAGE3`) that can be used to rebuild the /// lookup table. pub fn write_rust(&self, elem_ty: &str, mut fmt_elem: impl FnMut(&E) -> String) -> String { let mut out = String::new(); out.push_str("// This file is auto-generated. Do not edit.\n\n"); let _ = write!(out, "pub static STAGE1: [u16; {}] = [", self.stage1.len()); for e in &self.stage1 { let _ = write!(out, "{e},"); } out.push_str("];\n\n"); let _ = write!(out, "pub static STAGE2: [u16; {}] = [", self.stage2.len()); for e in &self.stage2 { let _ = write!(out, "{e},"); } out.push_str("];\n\n"); let _ = write!( out, "pub static STAGE3: [{elem_ty}; {}] = [", self.stage3.len() ); for e in &self.stage3 { let _ = write!(out, "{},", fmt_elem(e)); } out.push_str("];\n"); out } } /// A 3-level lookup table from a Unicode codepoint to a mapping of type `E`. #[derive(Debug, Clone, Copy)] pub struct Tables<'a, E> { pub stage1: &'a [u16], pub stage2: &'a [u16], pub stage3: &'a [E], } impl Tables<'_, E> { /// Given a codepoint (at most `0x1F_FFFF`), returns the mapping for it. #[inline] #[must_use] pub fn get(&self, cp: u32) -> E { let high = (cp >> 8) as usize; let low = (cp & 0xFF) as usize; self.stage3[self.stage2[self.stage1[high] as usize + low] as usize] } } #[cfg(test)] mod tests { use super::*; #[test] fn generate_roundtrip() { let f = |cp: u32| -> u8 { if cp < 0x80 { 0 } else if (0x1000..0x1100).contains(&cp) { 1 } else if cp == 0x1F600 { 2 } else { 3 } }; let t = generate(f).unwrap(); let tables = t.as_tables(); for cp in 0..CODEPOINT_COUNT as u32 { assert_eq!(tables.get(cp), f(cp), "cp={cp:x}"); } // Blocks are deduplicated. assert!(t.stage2.len() < 10 * BLOCK_SIZE); assert_eq!(t.stage1.len(), CODEPOINT_COUNT / BLOCK_SIZE); } #[test] fn write_rust_output() { let t = generate(|cp| cp == 0x41).unwrap(); let src = t.write_rust("bool", ToString::to_string); assert!(src.contains("pub static STAGE3: [bool; 2] = [false,true,];")); } }