feat(cli): port the synthetic data generator to Rust
Move RDG_genBuffer and RDG_genStdout into the Rust CLI helper archive while preserving the C generator's random-draw order, fixed-point literal table, sparse-data path, sliding dictionary, and public ABI. Keep a zero-length buffer a valid no-op even when its pointer is null. The focused tests include C reference vectors, sparse and pure-noise inputs, custom literal probabilities, reproducibility, and the null zero-length case. Test Plan: - cargo test --manifest-path rust/cli/Cargo.toml --no-default-features datagen::tests - cargo test --manifest-path rust/cli/Cargo.toml --all-features Refs: programs/datagen.c compatibility port
This commit is contained in:
@@ -0,0 +1,297 @@
|
||||
#![allow(non_camel_case_types)]
|
||||
#![allow(non_snake_case)]
|
||||
|
||||
//! Synthetic compressible-data generator for the command-line programs.
|
||||
//!
|
||||
//! Port of `programs/datagen.c`. For a given `(matchProba, litProba, seed)`
|
||||
//! triplet the output is byte-identical to the C generator: the CLI test
|
||||
//! suites rely on that determinism when they regenerate the same content in
|
||||
//! separate process invocations.
|
||||
//!
|
||||
//! The generator draws from a fixed-point literal-distribution table and a
|
||||
//! 15-bit match/literal decision stream; every random draw and integer cast
|
||||
//! below mirrors the C expression order exactly, because each draw advances
|
||||
//! the shared seed state.
|
||||
|
||||
use std::ffi::c_void;
|
||||
use std::io::Write;
|
||||
use std::os::raw::c_uint;
|
||||
|
||||
const LTLOG: u32 = 13;
|
||||
const LTSIZE: usize = 1 << LTLOG;
|
||||
const LTMASK: u32 = (LTSIZE as u32) - 1;
|
||||
|
||||
fn rdg_rand(src: &mut u32) -> u32 {
|
||||
const PRIME1: u32 = 2654435761;
|
||||
const PRIME2: u32 = 2246822519;
|
||||
let mut rand32 = *src;
|
||||
rand32 = rand32.wrapping_mul(PRIME1);
|
||||
rand32 ^= PRIME2;
|
||||
rand32 = rand32.rotate_left(13);
|
||||
*src = rand32;
|
||||
rand32 >> 5
|
||||
}
|
||||
|
||||
/// `fixedPoint_24_8` in the C source: a 24.8 fixed-point literal probability.
|
||||
fn rdg_fillLiteralDistrib(ldt: &mut [u8; LTSIZE], ld: u32) {
|
||||
let first_char: u8 = if ld == 0 { 0 } else { b'(' };
|
||||
let last_char: u8 = if ld == 0 { 255 } else { b'}' };
|
||||
let mut character: u8 = if ld == 0 { 0 } else { b'0' };
|
||||
|
||||
let mut u = 0usize;
|
||||
while u < LTSIZE {
|
||||
let weight = (((LTSIZE - u) as u32).wrapping_mul(ld) >> 8) + 1;
|
||||
let end = (u + weight as usize).min(LTSIZE);
|
||||
while u < end {
|
||||
ldt[u] = character;
|
||||
u += 1;
|
||||
}
|
||||
character = character.wrapping_add(1);
|
||||
if character > last_char {
|
||||
character = first_char;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn rdg_genChar(seed: &mut u32, ldt: &[u8; LTSIZE]) -> u8 {
|
||||
let id = rdg_rand(seed) & LTMASK;
|
||||
ldt[id as usize]
|
||||
}
|
||||
|
||||
fn rdg_rand15Bits(seed: &mut u32) -> u32 {
|
||||
rdg_rand(seed) & 0x7FFF
|
||||
}
|
||||
|
||||
fn rdg_randLength(seed: &mut u32) -> u32 {
|
||||
/* Two sequential draws, exactly as in C: the branch consumes one draw,
|
||||
* the length another. */
|
||||
if rdg_rand(seed) & 7 != 0 {
|
||||
rdg_rand(seed) & 0xF
|
||||
} else {
|
||||
(rdg_rand(seed) & 0x1FF) + 0xF
|
||||
}
|
||||
}
|
||||
|
||||
fn rdg_genBlock(
|
||||
buffer: &mut [u8],
|
||||
prefix_size: usize,
|
||||
match_proba: f64,
|
||||
ldt: &[u8; LTSIZE],
|
||||
seed: &mut u32,
|
||||
) {
|
||||
let buff_size = buffer.len();
|
||||
let match_proba32 = (32768.0 * match_proba) as u32;
|
||||
let mut pos = prefix_size;
|
||||
let mut prev_offset = 1u32;
|
||||
|
||||
/* special case : sparse content */
|
||||
if match_proba >= 1.0 {
|
||||
loop {
|
||||
let mut size0 = (rdg_rand(seed) & 3) as usize;
|
||||
size0 = 1usize << (16 + size0 * 2);
|
||||
size0 += (rdg_rand(seed) as usize) & (size0 - 1); /* because size0 is power of 2 */
|
||||
if buff_size < pos + size0 {
|
||||
buffer[pos..].fill(0);
|
||||
return;
|
||||
}
|
||||
buffer[pos..pos + size0].fill(0);
|
||||
pos += size0;
|
||||
buffer[pos - 1] = rdg_genChar(seed, ldt);
|
||||
}
|
||||
}
|
||||
|
||||
/* init */
|
||||
if pos == 0 {
|
||||
buffer[0] = rdg_genChar(seed, ldt);
|
||||
pos = 1;
|
||||
}
|
||||
|
||||
/* Generate compressible data */
|
||||
while pos < buff_size {
|
||||
/* Select : Literal (char) or Match (within 32K) */
|
||||
if rdg_rand15Bits(seed) < match_proba32 {
|
||||
/* Copy (within 32K). The C code narrows the end position to
|
||||
* U32; buffers handed to this generator are far below 4 GiB. */
|
||||
let length = rdg_randLength(seed) + 4;
|
||||
let d = (pos + length as usize).min(buff_size) as u32;
|
||||
let repeat_offset = (rdg_rand(seed) & 15) == 2;
|
||||
let rand_offset = rdg_rand15Bits(seed) + 1;
|
||||
let offset = if repeat_offset {
|
||||
prev_offset
|
||||
} else {
|
||||
/* Keep the C expression's size_t MIN before its U32 cast. */
|
||||
(rand_offset as usize).min(pos) as u32
|
||||
};
|
||||
let mut mtch = pos - offset as usize;
|
||||
while pos < d as usize {
|
||||
buffer[pos] = buffer[mtch]; /* correctly manages overlaps */
|
||||
pos += 1;
|
||||
mtch += 1;
|
||||
}
|
||||
prev_offset = offset;
|
||||
} else {
|
||||
/* Literal (noise) */
|
||||
let length = rdg_randLength(seed);
|
||||
let d = (pos + length as usize).min(buff_size) as u32;
|
||||
while pos < d as usize {
|
||||
buffer[pos] = rdg_genChar(seed, ldt);
|
||||
pos += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn build_distrib(match_proba: f64, mut lit_proba: f64) -> Box<[u8; LTSIZE]> {
|
||||
let mut ldt: Box<[u8; LTSIZE]> = Box::new([b'0'; LTSIZE]); /* yes, character '0', this is intentional */
|
||||
if lit_proba <= 0.0 {
|
||||
lit_proba = match_proba / 4.5;
|
||||
}
|
||||
rdg_fillLiteralDistrib(&mut ldt, (lit_proba * 256.0 + 0.001) as u32);
|
||||
ldt
|
||||
}
|
||||
|
||||
/// Rust implementation of the public `RDG_genBuffer()` ABI.
|
||||
///
|
||||
/// # Safety
|
||||
/// `buffer` must be valid for `size` writable bytes. It may be null when
|
||||
/// `size` is zero.
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn RDG_genBuffer(
|
||||
buffer: *mut c_void,
|
||||
size: usize,
|
||||
matchProba: f64,
|
||||
litProba: f64,
|
||||
seed: c_uint,
|
||||
) {
|
||||
if size == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut seed32 = seed;
|
||||
let ldt = build_distrib(matchProba, litProba);
|
||||
let buffer = unsafe { std::slice::from_raw_parts_mut(buffer.cast::<u8>(), size) };
|
||||
rdg_genBlock(buffer, 0, matchProba, &ldt, &mut seed32);
|
||||
}
|
||||
|
||||
/// Rust implementation of the public `RDG_genStdout()` ABI.
|
||||
#[no_mangle]
|
||||
pub extern "C" fn RDG_genStdout(size: u64, matchProba: f64, litProba: f64, seed: c_uint) {
|
||||
const STD_BLOCK_SIZE: usize = 128 * 1024;
|
||||
const STD_DICT_SIZE: usize = 32 * 1024;
|
||||
|
||||
let mut seed32 = seed;
|
||||
/* The C generator exits through perror()/exit(1) when its buffer cannot
|
||||
* be allocated; Rust's allocator aborts instead, which likewise cannot
|
||||
* be observed as data. */
|
||||
let mut buff = vec![0u8; STD_DICT_SIZE + STD_BLOCK_SIZE];
|
||||
let mut total: u64 = 0;
|
||||
let ldt = build_distrib(matchProba, litProba);
|
||||
|
||||
let stdout = std::io::stdout();
|
||||
let mut out = stdout.lock();
|
||||
|
||||
/* Generate initial dict */
|
||||
rdg_genBlock(&mut buff[..STD_DICT_SIZE], 0, matchProba, &ldt, &mut seed32);
|
||||
|
||||
/* Generate compressible data */
|
||||
while total < size {
|
||||
/* Match C's MIN in U64 before converting the result to size_t. */
|
||||
let gen_block_size = (STD_BLOCK_SIZE as u64).min(size - total) as usize;
|
||||
rdg_genBlock(&mut buff, STD_DICT_SIZE, matchProba, &ldt, &mut seed32);
|
||||
total += gen_block_size as u64;
|
||||
/* As in C, the write starts at the buffer base: the output is the
|
||||
* sliding window itself, and short-write errors are ignored. */
|
||||
let _ = out.write_all(&buff[..gen_block_size]);
|
||||
/* update dict */
|
||||
buff.copy_within(STD_BLOCK_SIZE.., 0);
|
||||
}
|
||||
|
||||
let _ = out.flush();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn gen(size: usize, match_proba: f64, lit_proba: f64, seed: u32) -> Vec<u8> {
|
||||
let mut buf = vec![0u8; size];
|
||||
unsafe { RDG_genBuffer(buf.as_mut_ptr().cast(), size, match_proba, lit_proba, seed) };
|
||||
buf
|
||||
}
|
||||
|
||||
/* Reference vectors produced by the original C implementation. */
|
||||
|
||||
#[test]
|
||||
fn matches_c_reference_at_default_probabilities() {
|
||||
assert_eq!(
|
||||
gen(64, 0.5, 0.0, 42),
|
||||
[
|
||||
0x45, 0x32, 0x32, 0x32, 0x34, 0x31, 0x34, 0x32, 0x35, 0x3b, 0x37, 0x33, 0x31, 0x36,
|
||||
0x32, 0x32, 0x33, 0x40, 0x37, 0x36, 0x32, 0x36, 0x42, 0x30, 0x55, 0x33, 0x43, 0x41,
|
||||
0x31, 0x3f, 0x3b, 0x31, 0x35, 0x46, 0x48, 0x32, 0x3a, 0x46, 0x33, 0x3f, 0x38, 0x35,
|
||||
0x3a, 0x40, 0x31, 0x32, 0x41, 0x3a, 0x55, 0x49, 0x3b, 0x32, 0x50, 0x39, 0x32, 0x32,
|
||||
0x33, 0x3c, 0x37, 0x31, 0x40, 0x30, 0x35, 0x32
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_c_reference_for_pure_noise() {
|
||||
assert_eq!(
|
||||
gen(64, 0.0, 0.0, 7),
|
||||
[
|
||||
0xd6, 0x86, 0x6d, 0x70, 0xbf, 0x0c, 0x36, 0x38, 0xbf, 0x90, 0xe4, 0xfb, 0x79, 0x96,
|
||||
0x7b, 0x33, 0x03, 0xb6, 0x73, 0x88, 0xd4, 0x21, 0x10, 0xcc, 0xb3, 0xf4, 0x2f, 0x5b,
|
||||
0x4f, 0x34, 0x7b, 0xb2, 0x1d, 0xe5, 0xc5, 0x2d, 0x37, 0x79, 0x03, 0x55, 0x1e, 0x0d,
|
||||
0x5e, 0xb2, 0x1c, 0x3e, 0xc1, 0x3a, 0x1a, 0x71, 0x68, 0x5d, 0x19, 0x24, 0xf9, 0x08,
|
||||
0xb0, 0x5f, 0x0a, 0x33, 0x02, 0xc1, 0xc9, 0xd3
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_c_reference_for_sparse_content() {
|
||||
assert_eq!(gen(64, 1.0, 0.0, 9), [0u8; 64]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_c_reference_with_custom_literal_probability() {
|
||||
assert_eq!(
|
||||
gen(64, 0.8, 0.3, 123),
|
||||
[
|
||||
0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
|
||||
0x30, 0x33, 0x34, 0x33, 0x30, 0x30, 0x36, 0x30, 0x32, 0x33, 0x31, 0x30, 0x31, 0x36,
|
||||
0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
|
||||
0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
|
||||
0x36, 0x36, 0x30, 0x33, 0x34, 0x33, 0x36, 0x36
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_seed_is_reproducible_and_seeds_differ() {
|
||||
assert_eq!(gen(512, 0.5, 0.0, 1), gen(512, 0.5, 0.0, 1));
|
||||
assert_ne!(gen(512, 0.5, 0.0, 1), gen(512, 0.5, 0.0, 2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_length_buffer_is_a_noop_even_with_a_null_pointer() {
|
||||
unsafe {
|
||||
RDG_genBuffer(std::ptr::null_mut(), 0, 0.5, 0.5, 0);
|
||||
}
|
||||
|
||||
let mut sentinel = [0xa5u8];
|
||||
unsafe {
|
||||
RDG_genBuffer(sentinel.as_mut_ptr().cast(), 0, 0.5, 0.5, 0);
|
||||
}
|
||||
assert_eq!(sentinel, [0xa5]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_literal_probability_fills_the_entire_distribution() {
|
||||
let ldt = build_distrib(0.0, 0.0);
|
||||
for (index, &character) in ldt.iter().enumerate() {
|
||||
assert_eq!(character, index as u8);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user