From 24d01b92fbdb290bf4ccae070318b53e244c2b62 Mon Sep 17 00:00:00 2001 From: ddidderr Date: Fri, 10 Jul 2026 21:28:56 +0200 Subject: [PATCH] feat(rust): port compression block pre-splitting Move ZSTD_splitBlock into Rust while retaining its caller-owned workspace contract. The implementation preserves the C fingerprint sampling heuristic and calls the migrated histogram primitive without adding a hot-path allocation. Reference-vector and randomized differential tests compare every split level with the original C translation unit. The C source remains as a declaration shim so existing source lists resolve the Rust ABI during the migration. Test Plan: - cargo fmt, cargo clippy, cargo clippy --benches, cargo clippy --tests - cargo test --all-targets and cargo build --release - cargo test/build --target i686-unknown-linux-gnu - 264-case original-C/Rust differential harness - C fuzzer, zstreamtest, and fuzzer32 smoke runs Refs: rust/README.md --- lib/compress/zstd_preSplit.c | 238 +--------------------- rust/README.md | 1 + rust/src/lib.rs | 1 + rust/src/zstd_presplit.rs | 376 +++++++++++++++++++++++++++++++++++ 4 files changed, 379 insertions(+), 237 deletions(-) create mode 100644 rust/src/zstd_presplit.rs diff --git a/lib/compress/zstd_preSplit.c b/lib/compress/zstd_preSplit.c index d820c20ac..ca7c34a71 100644 --- a/lib/compress/zstd_preSplit.c +++ b/lib/compress/zstd_preSplit.c @@ -1,238 +1,2 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#include "../common/compiler.h" /* ZSTD_ALIGNOF */ -#include "../common/mem.h" /* S64 */ -#include "../common/zstd_deps.h" /* ZSTD_memset */ -#include "../common/zstd_internal.h" /* ZSTD_STATIC_ASSERT */ -#include "hist.h" /* HIST_add */ +/* Implementation moved to rust/src/zstd_presplit.rs. */ #include "zstd_preSplit.h" - - -#define BLOCKSIZE_MIN 3500 -#define THRESHOLD_PENALTY_RATE 16 -#define THRESHOLD_BASE (THRESHOLD_PENALTY_RATE - 2) -#define THRESHOLD_PENALTY 3 - -#define HASHLENGTH 2 -#define HASHLOG_MAX 10 -#define HASHTABLESIZE (1 << HASHLOG_MAX) -#define HASHMASK (HASHTABLESIZE - 1) -#define KNUTH 0x9e3779b9 - -/* for hashLog > 8, hash 2 bytes. - * for hashLog == 8, just take the byte, no hashing. - * The speed of this method relies on compile-time constant propagation */ -FORCE_INLINE_TEMPLATE unsigned hash2(const void *p, unsigned hashLog) -{ - assert(hashLog >= 8); - if (hashLog == 8) return (U32)((const BYTE*)p)[0]; - assert(hashLog <= HASHLOG_MAX); - return (U32)(MEM_read16(p)) * KNUTH >> (32 - hashLog); -} - - -typedef struct { - unsigned events[HASHTABLESIZE]; - size_t nbEvents; -} Fingerprint; -typedef struct { - Fingerprint pastEvents; - Fingerprint newEvents; -} FPStats; - -static void initStats(FPStats* fpstats) -{ - ZSTD_memset(fpstats, 0, sizeof(FPStats)); -} - -FORCE_INLINE_TEMPLATE void -addEvents_generic(Fingerprint* fp, const void* src, size_t srcSize, size_t samplingRate, unsigned hashLog) -{ - const char* p = (const char*)src; - size_t limit = srcSize - HASHLENGTH + 1; - size_t n; - assert(srcSize >= HASHLENGTH); - for (n = 0; n < limit; n+=samplingRate) { - fp->events[hash2(p+n, hashLog)]++; - } - fp->nbEvents += limit/samplingRate; -} - -FORCE_INLINE_TEMPLATE void -recordFingerprint_generic(Fingerprint* fp, const void* src, size_t srcSize, size_t samplingRate, unsigned hashLog) -{ - ZSTD_memset(fp, 0, sizeof(unsigned) * ((size_t)1 << hashLog)); - fp->nbEvents = 0; - addEvents_generic(fp, src, srcSize, samplingRate, hashLog); -} - -typedef void (*RecordEvents_f)(Fingerprint* fp, const void* src, size_t srcSize); - -#define FP_RECORD(_rate) ZSTD_recordFingerprint_##_rate - -#define ZSTD_GEN_RECORD_FINGERPRINT(_rate, _hSize) \ - static void FP_RECORD(_rate)(Fingerprint* fp, const void* src, size_t srcSize) \ - { \ - recordFingerprint_generic(fp, src, srcSize, _rate, _hSize); \ - } - -ZSTD_GEN_RECORD_FINGERPRINT(1, 10) -ZSTD_GEN_RECORD_FINGERPRINT(5, 10) -ZSTD_GEN_RECORD_FINGERPRINT(11, 9) -ZSTD_GEN_RECORD_FINGERPRINT(43, 8) - - -static U64 abs64(S64 s64) { return (U64)((s64 < 0) ? -s64 : s64); } - -static U64 fpDistance(const Fingerprint* fp1, const Fingerprint* fp2, unsigned hashLog) -{ - U64 distance = 0; - size_t n; - assert(hashLog <= HASHLOG_MAX); - for (n = 0; n < ((size_t)1 << hashLog); n++) { - distance += - abs64((S64)fp1->events[n] * (S64)fp2->nbEvents - (S64)fp2->events[n] * (S64)fp1->nbEvents); - } - return distance; -} - -/* Compare newEvents with pastEvents - * return 1 when considered "too different" - */ -static int compareFingerprints(const Fingerprint* ref, - const Fingerprint* newfp, - int penalty, - unsigned hashLog) -{ - assert(ref->nbEvents > 0); - assert(newfp->nbEvents > 0); - { U64 p50 = (U64)ref->nbEvents * (U64)newfp->nbEvents; - U64 deviation = fpDistance(ref, newfp, hashLog); - U64 threshold = p50 * (U64)(THRESHOLD_BASE + penalty) / THRESHOLD_PENALTY_RATE; - return deviation >= threshold; - } -} - -static void mergeEvents(Fingerprint* acc, const Fingerprint* newfp) -{ - size_t n; - for (n = 0; n < HASHTABLESIZE; n++) { - acc->events[n] += newfp->events[n]; - } - acc->nbEvents += newfp->nbEvents; -} - -static void flushEvents(FPStats* fpstats) -{ - size_t n; - for (n = 0; n < HASHTABLESIZE; n++) { - fpstats->pastEvents.events[n] = fpstats->newEvents.events[n]; - } - fpstats->pastEvents.nbEvents = fpstats->newEvents.nbEvents; - ZSTD_memset(&fpstats->newEvents, 0, sizeof(fpstats->newEvents)); -} - -static void removeEvents(Fingerprint* acc, const Fingerprint* slice) -{ - size_t n; - for (n = 0; n < HASHTABLESIZE; n++) { - assert(acc->events[n] >= slice->events[n]); - acc->events[n] -= slice->events[n]; - } - acc->nbEvents -= slice->nbEvents; -} - -#define CHUNKSIZE (8 << 10) -static size_t ZSTD_splitBlock_byChunks(const void* blockStart, size_t blockSize, - int level, - void* workspace, size_t wkspSize) -{ - static const RecordEvents_f records_fs[] = { - FP_RECORD(43), FP_RECORD(11), FP_RECORD(5), FP_RECORD(1) - }; - static const unsigned hashParams[] = { 8, 9, 10, 10 }; - const RecordEvents_f record_f = (assert(0<=level && level<=3), records_fs[level]); - FPStats* const fpstats = (FPStats*)workspace; - const char* p = (const char*)blockStart; - int penalty = THRESHOLD_PENALTY; - size_t pos = 0; - assert(blockSize == (128 << 10)); - assert(workspace != NULL); - assert((size_t)workspace % ZSTD_ALIGNOF(FPStats) == 0); - ZSTD_STATIC_ASSERT(ZSTD_SLIPBLOCK_WORKSPACESIZE >= sizeof(FPStats)); - assert(wkspSize >= sizeof(FPStats)); (void)wkspSize; - - initStats(fpstats); - record_f(&fpstats->pastEvents, p, CHUNKSIZE); - for (pos = CHUNKSIZE; pos <= blockSize - CHUNKSIZE; pos += CHUNKSIZE) { - record_f(&fpstats->newEvents, p + pos, CHUNKSIZE); - if (compareFingerprints(&fpstats->pastEvents, &fpstats->newEvents, penalty, hashParams[level])) { - return pos; - } else { - mergeEvents(&fpstats->pastEvents, &fpstats->newEvents); - if (penalty > 0) penalty--; - } - } - assert(pos == blockSize); - return blockSize; - (void)flushEvents; (void)removeEvents; -} - -/* ZSTD_splitBlock_fromBorders(): very fast strategy : - * compare fingerprint from beginning and end of the block, - * derive from their difference if it's preferable to split in the middle, - * repeat the process a second time, for finer grained decision. - * 3 times did not brought improvements, so I stopped at 2. - * Benefits are good enough for a cheap heuristic. - * More accurate splitting saves more, but speed impact is also more perceptible. - * For better accuracy, use more elaborate variant *_byChunks. - */ -static size_t ZSTD_splitBlock_fromBorders(const void* blockStart, size_t blockSize, - void* workspace, size_t wkspSize) -{ -#define SEGMENT_SIZE 512 - FPStats* const fpstats = (FPStats*)workspace; - Fingerprint* middleEvents = (Fingerprint*)(void*)((char*)workspace + 512 * sizeof(unsigned)); - assert(blockSize == (128 << 10)); - assert(workspace != NULL); - assert((size_t)workspace % ZSTD_ALIGNOF(FPStats) == 0); - ZSTD_STATIC_ASSERT(ZSTD_SLIPBLOCK_WORKSPACESIZE >= sizeof(FPStats)); - assert(wkspSize >= sizeof(FPStats)); (void)wkspSize; - - initStats(fpstats); - HIST_add(fpstats->pastEvents.events, blockStart, SEGMENT_SIZE); - HIST_add(fpstats->newEvents.events, (const char*)blockStart + blockSize - SEGMENT_SIZE, SEGMENT_SIZE); - fpstats->pastEvents.nbEvents = fpstats->newEvents.nbEvents = SEGMENT_SIZE; - if (!compareFingerprints(&fpstats->pastEvents, &fpstats->newEvents, 0, 8)) - return blockSize; - - HIST_add(middleEvents->events, (const char*)blockStart + blockSize/2 - SEGMENT_SIZE/2, SEGMENT_SIZE); - middleEvents->nbEvents = SEGMENT_SIZE; - { U64 const distFromBegin = fpDistance(&fpstats->pastEvents, middleEvents, 8); - U64 const distFromEnd = fpDistance(&fpstats->newEvents, middleEvents, 8); - U64 const minDistance = SEGMENT_SIZE * SEGMENT_SIZE / 3; - if (abs64((S64)distFromBegin - (S64)distFromEnd) < minDistance) - return 64 KB; - return (distFromBegin > distFromEnd) ? 32 KB : 96 KB; - } -} - -size_t ZSTD_splitBlock(const void* blockStart, size_t blockSize, - int level, - void* workspace, size_t wkspSize) -{ - DEBUGLOG(6, "ZSTD_splitBlock (level=%i)", level); - assert(0<=level && level<=4); - if (level == 0) - return ZSTD_splitBlock_fromBorders(blockStart, blockSize, workspace, wkspSize); - /* level >= 1*/ - return ZSTD_splitBlock_byChunks(blockStart, blockSize, level-1, workspace, wkspSize); -} diff --git a/rust/README.md b/rust/README.md index 083353e3b..f19f4990f 100644 --- a/rust/README.md +++ b/rust/README.md @@ -26,6 +26,7 @@ zstd ABI: Huffman streams. - Compression primitives - `hist` counts byte frequencies for FSE and Huffman compression. + - `zstd_presplit` chooses split points for full compression blocks. - Runtime support - `threading` provides platform pthread wrappers required by zstd headers. - `pool` implements the bounded worker pool used by multithreaded compression. diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 288cbc342..7a8bffecf 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -17,3 +17,4 @@ pub mod threading; pub mod xxhash; pub mod zstd_common; pub mod zstd_ddict; +pub mod zstd_presplit; diff --git a/rust/src/zstd_presplit.rs b/rust/src/zstd_presplit.rs new file mode 100644 index 000000000..d80d3d085 --- /dev/null +++ b/rust/src/zstd_presplit.rs @@ -0,0 +1,376 @@ +#![allow(non_snake_case)] + +//! Compression block pre-splitting. +//! +//! This is the small distribution heuristic used by the compressor to decide +//! whether a full 128 KiB block should be split before match finding. It uses +//! caller-provided workspace exactly like the C implementation so it does not +//! add an allocation on the compression hot path. + +use crate::hist::HIST_add; +use crate::mem::MEM_read16; +use std::ffi::c_void; +use std::mem::size_of; +use std::os::raw::c_int; +use std::ptr; + +const BLOCK_SIZE: usize = 128 << 10; +const BLOCK_SIZE_MIN: usize = 3_500; +const THRESHOLD_PENALTY_RATE: u64 = 16; +const THRESHOLD_BASE: u64 = THRESHOLD_PENALTY_RATE - 2; +const THRESHOLD_PENALTY: u64 = 3; + +const HASH_LENGTH: usize = 2; +const HASH_LOG_MAX: u32 = 10; +const HASH_TABLE_SIZE: usize = 1 << HASH_LOG_MAX; +const KNUTH: u32 = 0x9e37_79b9; +const CHUNK_SIZE: usize = 8 << 10; +const SEGMENT_SIZE: usize = 512; +const WORKSPACE_SIZE: usize = 8_208; + +#[repr(C)] +struct Fingerprint { + events: [u32; HASH_TABLE_SIZE], + nb_events: usize, +} + +#[repr(C)] +struct FpStats { + past_events: Fingerprint, + new_events: Fingerprint, +} + +#[inline] +unsafe fn fingerprint_events(fp: *mut Fingerprint) -> *mut u32 { + unsafe { ptr::addr_of_mut!((*fp).events).cast() } +} + +#[inline] +unsafe fn fingerprint_nb_events(fp: *const Fingerprint) -> usize { + unsafe { ptr::addr_of!((*fp).nb_events).read() } +} + +#[inline] +unsafe fn set_fingerprint_nb_events(fp: *mut Fingerprint, value: usize) { + unsafe { ptr::addr_of_mut!((*fp).nb_events).write(value) } +} + +#[inline] +unsafe fn hash2(source: *const u8, hash_log: u32) -> usize { + debug_assert!((8..=HASH_LOG_MAX).contains(&hash_log)); + if hash_log == 8 { + unsafe { *source as usize } + } else { + let value = unsafe { MEM_read16(source.cast()) as u32 }; + (value.wrapping_mul(KNUTH) >> (32 - hash_log)) as usize + } +} + +unsafe fn init_stats(stats: *mut FpStats) { + unsafe { ptr::write_bytes(stats.cast::(), 0, size_of::()) } +} + +unsafe fn add_events( + fp: *mut Fingerprint, + source: *const u8, + source_size: usize, + sampling_rate: usize, + hash_log: u32, +) { + debug_assert!(source_size >= HASH_LENGTH); + let limit = source_size - HASH_LENGTH + 1; + let events = unsafe { fingerprint_events(fp) }; + let mut index = 0; + while index < limit { + let event = unsafe { events.add(hash2(source.add(index), hash_log)) }; + unsafe { event.write(event.read().wrapping_add(1)) }; + index += sampling_rate; + } + let nb_events = unsafe { fingerprint_nb_events(fp) }; + unsafe { set_fingerprint_nb_events(fp, nb_events.wrapping_add(limit / sampling_rate)) }; +} + +unsafe fn record_fingerprint( + fp: *mut Fingerprint, + source: *const u8, + source_size: usize, + sampling_rate: usize, + hash_log: u32, +) { + unsafe { + ptr::write_bytes(fingerprint_events(fp), 0, 1usize << hash_log); + set_fingerprint_nb_events(fp, 0); + add_events(fp, source, source_size, sampling_rate, hash_log); + } +} + +#[inline] +fn abs64(value: i64) -> u64 { + if value < 0 { + value.wrapping_neg() as u64 + } else { + value as u64 + } +} + +unsafe fn fp_distance(fp1: *const Fingerprint, fp2: *const Fingerprint, hash_log: u32) -> u64 { + debug_assert!(hash_log <= HASH_LOG_MAX); + let fp1_events = unsafe { fingerprint_events(fp1.cast_mut()) }; + let fp2_events = unsafe { fingerprint_events(fp2.cast_mut()) }; + let fp1_nb_events = unsafe { fingerprint_nb_events(fp1) } as i64; + let fp2_nb_events = unsafe { fingerprint_nb_events(fp2) } as i64; + let mut distance = 0u64; + for index in 0..(1usize << hash_log) { + let lhs = unsafe { fp1_events.add(index).read() as i64 } * fp2_nb_events; + let rhs = unsafe { fp2_events.add(index).read() as i64 } * fp1_nb_events; + distance = distance.wrapping_add(abs64(lhs.wrapping_sub(rhs))); + } + distance +} + +unsafe fn compare_fingerprints( + reference: *const Fingerprint, + new_fingerprint: *const Fingerprint, + penalty: u64, + hash_log: u32, +) -> bool { + let reference_events = unsafe { fingerprint_nb_events(reference) } as u64; + let new_events = unsafe { fingerprint_nb_events(new_fingerprint) } as u64; + debug_assert_ne!(reference_events, 0); + debug_assert_ne!(new_events, 0); + + let p50 = reference_events * new_events; + let deviation = unsafe { fp_distance(reference, new_fingerprint, hash_log) }; + let threshold = p50 * (THRESHOLD_BASE + penalty) / THRESHOLD_PENALTY_RATE; + deviation >= threshold +} + +unsafe fn merge_events(accumulator: *mut Fingerprint, new_fingerprint: *const Fingerprint) { + let accumulator_events = unsafe { fingerprint_events(accumulator) }; + let new_events = unsafe { fingerprint_events(new_fingerprint.cast_mut()) }; + for index in 0..HASH_TABLE_SIZE { + let value = unsafe { + accumulator_events + .add(index) + .read() + .wrapping_add(new_events.add(index).read()) + }; + unsafe { accumulator_events.add(index).write(value) }; + } + let nb_events = unsafe { fingerprint_nb_events(accumulator) } + .wrapping_add(unsafe { fingerprint_nb_events(new_fingerprint) }); + unsafe { set_fingerprint_nb_events(accumulator, nb_events) }; +} + +unsafe fn split_block_by_chunks( + block_start: *const u8, + block_size: usize, + level: usize, + workspace: *mut c_void, + workspace_size: usize, +) -> usize { + const RECORD_PARAMS: [(usize, u32); 4] = [(43, 8), (11, 9), (5, 10), (1, 10)]; + + debug_assert!(level <= 3); + debug_assert_eq!(block_size, BLOCK_SIZE); + debug_assert!(!workspace.is_null()); + debug_assert_eq!(workspace as usize % std::mem::align_of::(), 0); + debug_assert!(workspace_size >= size_of::()); + debug_assert!(workspace_size >= WORKSPACE_SIZE); + + let (sampling_rate, hash_log) = RECORD_PARAMS[level]; + let stats = workspace.cast::(); + let past_events = unsafe { ptr::addr_of_mut!((*stats).past_events) }; + let new_events = unsafe { ptr::addr_of_mut!((*stats).new_events) }; + unsafe { + init_stats(stats); + record_fingerprint( + past_events, + block_start, + CHUNK_SIZE, + sampling_rate, + hash_log, + ); + } + + let mut penalty = THRESHOLD_PENALTY; + let mut position = CHUNK_SIZE; + while position <= block_size - CHUNK_SIZE { + unsafe { + record_fingerprint( + new_events, + block_start.add(position), + CHUNK_SIZE, + sampling_rate, + hash_log, + ); + } + if unsafe { compare_fingerprints(past_events, new_events, penalty, hash_log) } { + return position; + } + unsafe { merge_events(past_events, new_events) }; + penalty = penalty.saturating_sub(1); + position += CHUNK_SIZE; + } + debug_assert_eq!(position, block_size); + block_size +} + +unsafe fn split_block_from_borders( + block_start: *const u8, + block_size: usize, + workspace: *mut c_void, + workspace_size: usize, +) -> usize { + debug_assert_eq!(block_size, BLOCK_SIZE); + debug_assert!(!workspace.is_null()); + debug_assert_eq!(workspace as usize % std::mem::align_of::(), 0); + debug_assert!(workspace_size >= size_of::()); + debug_assert!(workspace_size >= WORKSPACE_SIZE); + + let stats = workspace.cast::(); + let past_events = unsafe { ptr::addr_of_mut!((*stats).past_events) }; + let new_events = unsafe { ptr::addr_of_mut!((*stats).new_events) }; + let middle_events = unsafe { workspace.cast::().add(SEGMENT_SIZE * size_of::()) } + .cast::(); + + unsafe { + init_stats(stats); + HIST_add( + fingerprint_events(past_events), + block_start.cast(), + SEGMENT_SIZE, + ); + HIST_add( + fingerprint_events(new_events), + block_start.add(block_size - SEGMENT_SIZE).cast(), + SEGMENT_SIZE, + ); + set_fingerprint_nb_events(past_events, SEGMENT_SIZE); + set_fingerprint_nb_events(new_events, SEGMENT_SIZE); + } + if !unsafe { compare_fingerprints(past_events, new_events, 0, 8) } { + return block_size; + } + + unsafe { + HIST_add( + fingerprint_events(middle_events), + block_start.add(block_size / 2 - SEGMENT_SIZE / 2).cast(), + SEGMENT_SIZE, + ); + set_fingerprint_nb_events(middle_events, SEGMENT_SIZE); + } + let distance_from_begin = unsafe { fp_distance(past_events, middle_events, 8) }; + let distance_from_end = unsafe { fp_distance(new_events, middle_events, 8) }; + let minimum_distance = (SEGMENT_SIZE * SEGMENT_SIZE / 3) as u64; + if abs64((distance_from_begin as i64).wrapping_sub(distance_from_end as i64)) < minimum_distance + { + 64 << 10 + } else if distance_from_begin > distance_from_end { + 32 << 10 + } else { + 96 << 10 + } +} + +/// Chooses a split point for a full-size compression block. +/// +/// # Safety +/// `block_start` must reference exactly one readable 128 KiB block. `workspace` +/// must be aligned for `FpStats` and provide at least 8,208 bytes. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_splitBlock( + block_start: *const c_void, + block_size: usize, + level: c_int, + workspace: *mut c_void, + workspace_size: usize, +) -> usize { + debug_assert!((0..=4).contains(&level)); + debug_assert!(block_size >= BLOCK_SIZE_MIN); + if level == 0 { + unsafe { + split_block_from_borders(block_start.cast(), block_size, workspace, workspace_size) + } + } else { + unsafe { + split_block_by_chunks( + block_start.cast(), + block_size, + (level - 1) as usize, + workspace, + workspace_size, + ) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn split(input: &[u8], level: c_int) -> usize { + let mut workspace = vec![0usize; WORKSPACE_SIZE.div_ceil(size_of::())]; + unsafe { + ZSTD_splitBlock( + input.as_ptr().cast(), + input.len(), + level, + workspace.as_mut_ptr().cast(), + workspace.len() * size_of::(), + ) + } + } + + fn assert_all_levels(input: &[u8], expected: [usize; 5]) { + for (level, expected) in expected.into_iter().enumerate() { + assert_eq!(split(input, level as c_int), expected, "level {level}"); + } + } + + #[test] + fn workspace_layout_fits_the_public_contract() { + assert!(size_of::() <= WORKSPACE_SIZE); + assert_eq!( + size_of::(), + HASH_TABLE_SIZE * size_of::() + size_of::() + ); + } + + #[test] + fn split_points_match_c_reference_vectors() { + let mut input = vec![0u8; BLOCK_SIZE]; + assert_all_levels(&input, [BLOCK_SIZE; 5]); + + input[..BLOCK_SIZE / 2].fill(0); + input[BLOCK_SIZE / 2..].fill(1); + assert_all_levels(&input, [64 << 10; 5]); + + input[..BLOCK_SIZE / 4].fill(0); + input[BLOCK_SIZE / 4..].fill(1); + assert_all_levels(&input, [32 << 10; 5]); + } + + #[test] + fn sampled_levels_match_c_reference_vectors() { + let mut input = vec![0u8; BLOCK_SIZE]; + let mut seed = 0x1234_5678u32; + for byte in &mut input { + seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + *byte = (seed >> 24) as u8; + } + assert_all_levels( + &input, + [BLOCK_SIZE, CHUNK_SIZE, BLOCK_SIZE, BLOCK_SIZE, BLOCK_SIZE], + ); + + for (index, byte) in input.iter_mut().enumerate() { + *byte = ((index / CHUNK_SIZE) & 1) as u8; + } + assert_all_levels( + &input, + [64 << 10, CHUNK_SIZE, CHUNK_SIZE, CHUNK_SIZE, CHUNK_SIZE], + ); + } +}