diff --git a/rust/src/datagen.rs b/rust/src/datagen.rs new file mode 100644 index 000000000..908c6869d --- /dev/null +++ b/rust/src/datagen.rs @@ -0,0 +1,297 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] + +//! Synthetic compressible-data generator for the command-line programs. +//! +//! Port of `programs/datagen.c`. For a given `(matchProba, litProba, seed)` +//! triplet the output is byte-identical to the C generator: the CLI test +//! suites rely on that determinism when they regenerate the same content in +//! separate process invocations. +//! +//! The generator draws from a fixed-point literal-distribution table and a +//! 15-bit match/literal decision stream; every random draw and integer cast +//! below mirrors the C expression order exactly, because each draw advances +//! the shared seed state. + +use std::ffi::c_void; +use std::io::Write; +use std::os::raw::c_uint; + +const LTLOG: u32 = 13; +const LTSIZE: usize = 1 << LTLOG; +const LTMASK: u32 = (LTSIZE as u32) - 1; + +fn rdg_rand(src: &mut u32) -> u32 { + const PRIME1: u32 = 2654435761; + const PRIME2: u32 = 2246822519; + let mut rand32 = *src; + rand32 = rand32.wrapping_mul(PRIME1); + rand32 ^= PRIME2; + rand32 = rand32.rotate_left(13); + *src = rand32; + rand32 >> 5 +} + +/// `fixedPoint_24_8` in the C source: a 24.8 fixed-point literal probability. +fn rdg_fillLiteralDistrib(ldt: &mut [u8; LTSIZE], ld: u32) { + let first_char: u8 = if ld == 0 { 0 } else { b'(' }; + let last_char: u8 = if ld == 0 { 255 } else { b'}' }; + let mut character: u8 = if ld == 0 { 0 } else { b'0' }; + + let mut u = 0usize; + while u < LTSIZE { + let weight = (((LTSIZE - u) as u32).wrapping_mul(ld) >> 8) + 1; + let end = (u + weight as usize).min(LTSIZE); + while u < end { + ldt[u] = character; + u += 1; + } + character = character.wrapping_add(1); + if character > last_char { + character = first_char; + } + } +} + +fn rdg_genChar(seed: &mut u32, ldt: &[u8; LTSIZE]) -> u8 { + let id = rdg_rand(seed) & LTMASK; + ldt[id as usize] +} + +fn rdg_rand15Bits(seed: &mut u32) -> u32 { + rdg_rand(seed) & 0x7FFF +} + +fn rdg_randLength(seed: &mut u32) -> u32 { + /* Two sequential draws, exactly as in C: the branch consumes one draw, + * the length another. */ + if rdg_rand(seed) & 7 != 0 { + rdg_rand(seed) & 0xF + } else { + (rdg_rand(seed) & 0x1FF) + 0xF + } +} + +fn rdg_genBlock( + buffer: &mut [u8], + prefix_size: usize, + match_proba: f64, + ldt: &[u8; LTSIZE], + seed: &mut u32, +) { + let buff_size = buffer.len(); + let match_proba32 = (32768.0 * match_proba) as u32; + let mut pos = prefix_size; + let mut prev_offset = 1u32; + + /* special case : sparse content */ + if match_proba >= 1.0 { + loop { + let mut size0 = (rdg_rand(seed) & 3) as usize; + size0 = 1usize << (16 + size0 * 2); + size0 += (rdg_rand(seed) as usize) & (size0 - 1); /* because size0 is power of 2 */ + if buff_size < pos + size0 { + buffer[pos..].fill(0); + return; + } + buffer[pos..pos + size0].fill(0); + pos += size0; + buffer[pos - 1] = rdg_genChar(seed, ldt); + } + } + + /* init */ + if pos == 0 { + buffer[0] = rdg_genChar(seed, ldt); + pos = 1; + } + + /* Generate compressible data */ + while pos < buff_size { + /* Select : Literal (char) or Match (within 32K) */ + if rdg_rand15Bits(seed) < match_proba32 { + /* Copy (within 32K). The C code narrows the end position to + * U32; buffers handed to this generator are far below 4 GiB. */ + let length = rdg_randLength(seed) + 4; + let d = (pos + length as usize).min(buff_size) as u32; + let repeat_offset = (rdg_rand(seed) & 15) == 2; + let rand_offset = rdg_rand15Bits(seed) + 1; + let offset = if repeat_offset { + prev_offset + } else { + /* Keep the C expression's size_t MIN before its U32 cast. */ + (rand_offset as usize).min(pos) as u32 + }; + let mut mtch = pos - offset as usize; + while pos < d as usize { + buffer[pos] = buffer[mtch]; /* correctly manages overlaps */ + pos += 1; + mtch += 1; + } + prev_offset = offset; + } else { + /* Literal (noise) */ + let length = rdg_randLength(seed); + let d = (pos + length as usize).min(buff_size) as u32; + while pos < d as usize { + buffer[pos] = rdg_genChar(seed, ldt); + pos += 1; + } + } + } +} + +fn build_distrib(match_proba: f64, mut lit_proba: f64) -> Box<[u8; LTSIZE]> { + let mut ldt: Box<[u8; LTSIZE]> = Box::new([b'0'; LTSIZE]); /* yes, character '0', this is intentional */ + if lit_proba <= 0.0 { + lit_proba = match_proba / 4.5; + } + rdg_fillLiteralDistrib(&mut ldt, (lit_proba * 256.0 + 0.001) as u32); + ldt +} + +/// Rust implementation of the public `RDG_genBuffer()` ABI. +/// +/// # Safety +/// `buffer` must be valid for `size` writable bytes. It may be null when +/// `size` is zero. +#[no_mangle] +pub unsafe extern "C" fn RDG_genBuffer( + buffer: *mut c_void, + size: usize, + matchProba: f64, + litProba: f64, + seed: c_uint, +) { + if size == 0 { + return; + } + + let mut seed32 = seed; + let ldt = build_distrib(matchProba, litProba); + let buffer = unsafe { std::slice::from_raw_parts_mut(buffer.cast::(), size) }; + rdg_genBlock(buffer, 0, matchProba, &ldt, &mut seed32); +} + +/// Rust implementation of the public `RDG_genStdout()` ABI. +#[no_mangle] +pub extern "C" fn RDG_genStdout(size: u64, matchProba: f64, litProba: f64, seed: c_uint) { + const STD_BLOCK_SIZE: usize = 128 * 1024; + const STD_DICT_SIZE: usize = 32 * 1024; + + let mut seed32 = seed; + /* The C generator exits through perror()/exit(1) when its buffer cannot + * be allocated; Rust's allocator aborts instead, which likewise cannot + * be observed as data. */ + let mut buff = vec![0u8; STD_DICT_SIZE + STD_BLOCK_SIZE]; + let mut total: u64 = 0; + let ldt = build_distrib(matchProba, litProba); + + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + + /* Generate initial dict */ + rdg_genBlock(&mut buff[..STD_DICT_SIZE], 0, matchProba, &ldt, &mut seed32); + + /* Generate compressible data */ + while total < size { + /* Match C's MIN in U64 before converting the result to size_t. */ + let gen_block_size = (STD_BLOCK_SIZE as u64).min(size - total) as usize; + rdg_genBlock(&mut buff, STD_DICT_SIZE, matchProba, &ldt, &mut seed32); + total += gen_block_size as u64; + /* As in C, the write starts at the buffer base: the output is the + * sliding window itself, and short-write errors are ignored. */ + let _ = out.write_all(&buff[..gen_block_size]); + /* update dict */ + buff.copy_within(STD_BLOCK_SIZE.., 0); + } + + let _ = out.flush(); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn gen(size: usize, match_proba: f64, lit_proba: f64, seed: u32) -> Vec { + let mut buf = vec![0u8; size]; + unsafe { RDG_genBuffer(buf.as_mut_ptr().cast(), size, match_proba, lit_proba, seed) }; + buf + } + + /* Reference vectors produced by the original C implementation. */ + + #[test] + fn matches_c_reference_at_default_probabilities() { + assert_eq!( + gen(64, 0.5, 0.0, 42), + [ + 0x45, 0x32, 0x32, 0x32, 0x34, 0x31, 0x34, 0x32, 0x35, 0x3b, 0x37, 0x33, 0x31, 0x36, + 0x32, 0x32, 0x33, 0x40, 0x37, 0x36, 0x32, 0x36, 0x42, 0x30, 0x55, 0x33, 0x43, 0x41, + 0x31, 0x3f, 0x3b, 0x31, 0x35, 0x46, 0x48, 0x32, 0x3a, 0x46, 0x33, 0x3f, 0x38, 0x35, + 0x3a, 0x40, 0x31, 0x32, 0x41, 0x3a, 0x55, 0x49, 0x3b, 0x32, 0x50, 0x39, 0x32, 0x32, + 0x33, 0x3c, 0x37, 0x31, 0x40, 0x30, 0x35, 0x32 + ] + ); + } + + #[test] + fn matches_c_reference_for_pure_noise() { + assert_eq!( + gen(64, 0.0, 0.0, 7), + [ + 0xd6, 0x86, 0x6d, 0x70, 0xbf, 0x0c, 0x36, 0x38, 0xbf, 0x90, 0xe4, 0xfb, 0x79, 0x96, + 0x7b, 0x33, 0x03, 0xb6, 0x73, 0x88, 0xd4, 0x21, 0x10, 0xcc, 0xb3, 0xf4, 0x2f, 0x5b, + 0x4f, 0x34, 0x7b, 0xb2, 0x1d, 0xe5, 0xc5, 0x2d, 0x37, 0x79, 0x03, 0x55, 0x1e, 0x0d, + 0x5e, 0xb2, 0x1c, 0x3e, 0xc1, 0x3a, 0x1a, 0x71, 0x68, 0x5d, 0x19, 0x24, 0xf9, 0x08, + 0xb0, 0x5f, 0x0a, 0x33, 0x02, 0xc1, 0xc9, 0xd3 + ] + ); + } + + #[test] + fn matches_c_reference_for_sparse_content() { + assert_eq!(gen(64, 1.0, 0.0, 9), [0u8; 64]); + } + + #[test] + fn matches_c_reference_with_custom_literal_probability() { + assert_eq!( + gen(64, 0.8, 0.3, 123), + [ + 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, + 0x30, 0x33, 0x34, 0x33, 0x30, 0x30, 0x36, 0x30, 0x32, 0x33, 0x31, 0x30, 0x31, 0x36, + 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, + 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, + 0x36, 0x36, 0x30, 0x33, 0x34, 0x33, 0x36, 0x36 + ] + ); + } + + #[test] + fn same_seed_is_reproducible_and_seeds_differ() { + assert_eq!(gen(512, 0.5, 0.0, 1), gen(512, 0.5, 0.0, 1)); + assert_ne!(gen(512, 0.5, 0.0, 1), gen(512, 0.5, 0.0, 2)); + } + + #[test] + fn zero_length_buffer_is_a_noop_even_with_a_null_pointer() { + unsafe { + RDG_genBuffer(std::ptr::null_mut(), 0, 0.5, 0.5, 0); + } + + let mut sentinel = [0xa5u8]; + unsafe { + RDG_genBuffer(sentinel.as_mut_ptr().cast(), 0, 0.5, 0.5, 0); + } + assert_eq!(sentinel, [0xa5]); + } + + #[test] + fn zero_literal_probability_fills_the_entire_distribution() { + let ldt = build_distrib(0.0, 0.0); + for (index, &character) in ldt.iter().enumerate() { + assert_eq!(character, index as u8); + } + } +}