diff --git a/lib/compress/hist.c b/lib/compress/hist.c index 4ccf9a90a..95267d1be 100644 --- a/lib/compress/hist.c +++ b/lib/compress/hist.c @@ -1,191 +1,3 @@ -/* ****************************************************************** - * hist : Histogram functions - * part of Finite State Entropy project - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * You can contact the author at : - * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy - * - Public forum : https://groups.google.com/forum/#!forum/lz4c - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. -****************************************************************** */ - -/* --- dependencies --- */ -#include "../common/mem.h" /* U32, BYTE, etc. */ -#include "../common/debug.h" /* assert, DEBUGLOG */ -#include "../common/error_private.h" /* ERROR */ #include "hist.h" - -/* --- Error management --- */ -unsigned HIST_isError(size_t code) { return ERR_isError(code); } - -/*-************************************************************** - * Histogram functions - ****************************************************************/ -void HIST_add(unsigned* count, const void* src, size_t srcSize) -{ - const BYTE* ip = (const BYTE*)src; - const BYTE* const end = ip + srcSize; - - while (ip largestCount) largestCount = count[s]; - } - - return largestCount; -} - -typedef enum { trustInput, checkMaxSymbolValue } HIST_checkInput_e; - -/* HIST_count_parallel_wksp() : - * store histogram into 4 intermediate tables, recombined at the end. - * this design makes better use of OoO cpus, - * and is noticeably faster when some values are heavily repeated. - * But it needs some additional workspace for intermediate tables. - * `workSpace` must be a U32 table of size >= HIST_WKSP_SIZE_U32. - * @return : largest histogram frequency, - * or an error code (notably when histogram's alphabet is larger than *maxSymbolValuePtr) */ -static size_t HIST_count_parallel_wksp( - unsigned* count, unsigned* maxSymbolValuePtr, - const void* source, size_t sourceSize, - HIST_checkInput_e check, - U32* const workSpace) -{ - const BYTE* ip = (const BYTE*)source; - const BYTE* const iend = ip+sourceSize; - size_t const countSize = (*maxSymbolValuePtr + 1) * sizeof(*count); - unsigned max=0; - U32* const Counting1 = workSpace; - U32* const Counting2 = Counting1 + 256; - U32* const Counting3 = Counting2 + 256; - U32* const Counting4 = Counting3 + 256; - - /* safety checks */ - assert(*maxSymbolValuePtr <= 255); - if (!sourceSize) { - ZSTD_memset(count, 0, countSize); - *maxSymbolValuePtr = 0; - return 0; - } - ZSTD_memset(workSpace, 0, 4*256*sizeof(unsigned)); - - /* by stripes of 16 bytes */ - { U32 cached = MEM_read32(ip); ip += 4; - while (ip < iend-15) { - U32 c = cached; cached = MEM_read32(ip); ip += 4; - Counting1[(BYTE) c ]++; - Counting2[(BYTE)(c>>8) ]++; - Counting3[(BYTE)(c>>16)]++; - Counting4[ c>>24 ]++; - c = cached; cached = MEM_read32(ip); ip += 4; - Counting1[(BYTE) c ]++; - Counting2[(BYTE)(c>>8) ]++; - Counting3[(BYTE)(c>>16)]++; - Counting4[ c>>24 ]++; - c = cached; cached = MEM_read32(ip); ip += 4; - Counting1[(BYTE) c ]++; - Counting2[(BYTE)(c>>8) ]++; - Counting3[(BYTE)(c>>16)]++; - Counting4[ c>>24 ]++; - c = cached; cached = MEM_read32(ip); ip += 4; - Counting1[(BYTE) c ]++; - Counting2[(BYTE)(c>>8) ]++; - Counting3[(BYTE)(c>>16)]++; - Counting4[ c>>24 ]++; - } - ip-=4; - } - - /* finish last symbols */ - while (ip max) max = Counting1[s]; - } } - - { unsigned maxSymbolValue = 255; - while (!Counting1[maxSymbolValue]) maxSymbolValue--; - if (check && maxSymbolValue > *maxSymbolValuePtr) return ERROR(maxSymbolValue_tooSmall); - *maxSymbolValuePtr = maxSymbolValue; - ZSTD_memmove(count, Counting1, countSize); /* in case count & Counting1 are overlapping */ - } - return (size_t)max; -} - -/* HIST_countFast_wksp() : - * Same as HIST_countFast(), but using an externally provided scratch buffer. - * `workSpace` is a writable buffer which must be 4-bytes aligned, - * `workSpaceSize` must be >= HIST_WKSP_SIZE - */ -size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, - const void* source, size_t sourceSize, - void* workSpace, size_t workSpaceSize) -{ - if (sourceSize < 1500) /* heuristic threshold */ - return HIST_count_simple(count, maxSymbolValuePtr, source, sourceSize); - if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ - if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall); - return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, trustInput, (U32*)workSpace); -} - -/* HIST_count_wksp() : - * Same as HIST_count(), but using an externally provided scratch buffer. - * `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */ -size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, - const void* source, size_t sourceSize, - void* workSpace, size_t workSpaceSize) -{ - if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ - if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall); - if (*maxSymbolValuePtr < 255) - return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, checkMaxSymbolValue, (U32*)workSpace); - *maxSymbolValuePtr = 255; - return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize); -} - -#ifndef ZSTD_NO_UNUSED_FUNCTIONS -/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */ -size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr, - const void* source, size_t sourceSize) -{ - unsigned tmpCounters[HIST_WKSP_SIZE_U32]; - return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters)); -} - -size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr, - const void* src, size_t srcSize) -{ - unsigned tmpCounters[HIST_WKSP_SIZE_U32]; - return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters)); -} -#endif +/* Implementation moved to rust/src/hist.rs. */ diff --git a/rust/README.md b/rust/README.md index 2f92fd302..be6bb8fe8 100644 --- a/rust/README.md +++ b/rust/README.md @@ -20,14 +20,16 @@ zstd ABI: - Entropy coding - `entropy_common` reads FSE normalized counts and Huffman statistics. - `fse_decompress` builds FSE decoding tables and decodes FSE streams. +- Compression primitives + - `hist` counts byte frequencies for FSE and Huffman compression. - Runtime support - `threading` provides platform pthread wrappers required by zstd headers. - `pool` implements the bounded worker pool used by multithreaded compression. -The remaining compression, general decompression, dictionary, legacy, and CLI -translation units are still C. They must move before the rewrite is complete. -Keeping that boundary explicit prevents a passing hybrid build from being -mistaken for the final all-Rust result. +The remaining block compression, general decompression, dictionary, legacy, +and CLI translation units are still C. They must move before the rewrite is +complete. Keeping that boundary explicit prevents a passing hybrid build from +being mistaken for the final all-Rust result. ## Compatibility boundary @@ -56,8 +58,8 @@ Then run original compatibility tests from the repository root, starting with the narrow target for the component being migrated. For example: ```sh -make -C tests poolTests -./tests/poolTests +make -C tests fuzzer +./tests/fuzzer -i1 --no-big-tests ``` Broader `tests/Makefile` targets remain the authoritative integration gates as diff --git a/rust/src/hist.rs b/rust/src/hist.rs new file mode 100644 index 000000000..2bc8ae0f2 --- /dev/null +++ b/rust/src/hist.rs @@ -0,0 +1,372 @@ +#![allow(non_snake_case)] + +use crate::errors::{ERR_isError, ZstdErrorCode, ERROR}; +use std::ffi::c_void; +use std::os::raw::c_uint; + +pub const HIST_WKSP_SIZE_U32: usize = 1024; +pub const HIST_WKSP_SIZE: usize = HIST_WKSP_SIZE_U32 * std::mem::size_of::(); + +#[no_mangle] +pub extern "C" fn HIST_isError(code: usize) -> c_uint { + ERR_isError(code) as c_uint +} + +#[no_mangle] +pub unsafe extern "C" fn HIST_add(count: *mut c_uint, src: *const c_void, src_size: usize) { + let mut ip = src.cast::(); + let iend = ip.wrapping_add(src_size); + while ip < iend { + let counter = count.add(*ip as usize); + *counter = (*counter).wrapping_add(1); + ip = ip.add(1); + } +} + +#[no_mangle] +pub unsafe extern "C" fn HIST_count_simple( + count: *mut c_uint, + max_symbol_value_ptr: *mut c_uint, + src: *const c_void, + src_size: usize, +) -> c_uint { + let mut max_symbol_value = *max_symbol_value_ptr; + std::ptr::write_bytes(count, 0, max_symbol_value as usize + 1); + + if src_size == 0 { + *max_symbol_value_ptr = 0; + return 0; + } + + let mut ip = src.cast::(); + let iend = ip.add(src_size); + while ip < iend { + let counter = count.add(*ip as usize); + *counter = (*counter).wrapping_add(1); + ip = ip.add(1); + } + + while *count.add(max_symbol_value as usize) == 0 { + max_symbol_value -= 1; + } + *max_symbol_value_ptr = max_symbol_value; + + let mut largest_count = 0; + for symbol in 0..=max_symbol_value as usize { + largest_count = largest_count.max(*count.add(symbol)); + } + largest_count +} + +unsafe fn HIST_count_parallel_wksp( + count: *mut c_uint, + max_symbol_value_ptr: *mut c_uint, + source: *const c_void, + source_size: usize, + check_max_symbol_value: bool, + workspace: *mut c_uint, +) -> usize { + let requested_max = *max_symbol_value_ptr as usize; + let count_size = requested_max + 1; + + if source_size == 0 { + std::ptr::write_bytes(count, 0, count_size); + *max_symbol_value_ptr = 0; + return 0; + } + + std::ptr::write_bytes(workspace, 0, HIST_WKSP_SIZE_U32); + let counting1 = workspace; + let counting2 = workspace.add(256); + let counting3 = workspace.add(512); + let counting4 = workspace.add(768); + + let source = source.cast::(); + let mut offset = 0; + + // Keep at least four bytes for the scalar tail, matching the C loop's + // pipelined 16-byte stripes without its intentional short-input over-read. + while source_size - offset >= 20 { + for word in 0..4 { + let bytes = source.add(offset + word * 4); + let counters = [counting1, counting2, counting3, counting4]; + for (byte, counting) in counters.into_iter().enumerate() { + let counter = counting.add(*bytes.add(byte) as usize); + *counter = (*counter).wrapping_add(1); + } + } + offset += 16; + } + + while offset < source_size { + let counter = counting1.add(*source.add(offset) as usize); + *counter = (*counter).wrapping_add(1); + offset += 1; + } + + let mut largest_count = 0; + for symbol in 0..256 { + let total = (*counting1.add(symbol)) + .wrapping_add(*counting2.add(symbol)) + .wrapping_add(*counting3.add(symbol)) + .wrapping_add(*counting4.add(symbol)); + *counting1.add(symbol) = total; + largest_count = largest_count.max(total); + } + + let mut max_symbol_value = 255usize; + while *counting1.add(max_symbol_value) == 0 { + max_symbol_value -= 1; + } + if check_max_symbol_value && max_symbol_value > requested_max { + return ERROR(ZstdErrorCode::MaxSymbolValueTooSmall); + } + + *max_symbol_value_ptr = max_symbol_value as c_uint; + std::ptr::copy(counting1, count, count_size); + largest_count as usize +} + +#[no_mangle] +pub unsafe extern "C" fn HIST_countFast_wksp( + count: *mut c_uint, + max_symbol_value_ptr: *mut c_uint, + source: *const c_void, + source_size: usize, + workspace: *mut c_void, + workspace_size: usize, +) -> usize { + if source_size < 1500 { + return HIST_count_simple(count, max_symbol_value_ptr, source, source_size) as usize; + } + if workspace as usize & 3 != 0 { + return ERROR(ZstdErrorCode::Generic); + } + if workspace_size < HIST_WKSP_SIZE { + return ERROR(ZstdErrorCode::WorkSpaceTooSmall); + } + HIST_count_parallel_wksp( + count, + max_symbol_value_ptr, + source, + source_size, + false, + workspace.cast(), + ) +} + +#[no_mangle] +pub unsafe extern "C" fn HIST_count_wksp( + count: *mut c_uint, + max_symbol_value_ptr: *mut c_uint, + source: *const c_void, + source_size: usize, + workspace: *mut c_void, + workspace_size: usize, +) -> usize { + if workspace as usize & 3 != 0 { + return ERROR(ZstdErrorCode::Generic); + } + if workspace_size < HIST_WKSP_SIZE { + return ERROR(ZstdErrorCode::WorkSpaceTooSmall); + } + if *max_symbol_value_ptr < 255 { + return HIST_count_parallel_wksp( + count, + max_symbol_value_ptr, + source, + source_size, + true, + workspace.cast(), + ); + } + + *max_symbol_value_ptr = 255; + HIST_countFast_wksp( + count, + max_symbol_value_ptr, + source, + source_size, + workspace, + workspace_size, + ) +} + +#[no_mangle] +pub unsafe extern "C" fn HIST_countFast( + count: *mut c_uint, + max_symbol_value_ptr: *mut c_uint, + source: *const c_void, + source_size: usize, +) -> usize { + let mut workspace = [0u32; HIST_WKSP_SIZE_U32]; + HIST_countFast_wksp( + count, + max_symbol_value_ptr, + source, + source_size, + workspace.as_mut_ptr().cast(), + HIST_WKSP_SIZE, + ) +} + +#[no_mangle] +pub unsafe extern "C" fn HIST_count( + count: *mut c_uint, + max_symbol_value_ptr: *mut c_uint, + source: *const c_void, + source_size: usize, +) -> usize { + let mut workspace = [0u32; HIST_WKSP_SIZE_U32]; + HIST_count_wksp( + count, + max_symbol_value_ptr, + source, + source_size, + workspace.as_mut_ptr().cast(), + HIST_WKSP_SIZE, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn add_accumulates_counts() { + let source = [0u8, 1, 1, 3, 3, 3]; + let mut counts = [0u32; 256]; + counts[1] = 4; + + unsafe { HIST_add(counts.as_mut_ptr(), source.as_ptr().cast(), source.len()) }; + + assert_eq!(counts[0], 1); + assert_eq!(counts[1], 6); + assert_eq!(counts[2], 0); + assert_eq!(counts[3], 3); + } + + #[test] + fn simple_count_reports_largest_count_and_symbol() { + let source = [0u8, 3, 3, 2, 3, 2]; + let mut counts = [99u32; 256]; + let mut max_symbol = 255; + + let largest = unsafe { + HIST_count_simple( + counts.as_mut_ptr(), + &mut max_symbol, + source.as_ptr().cast(), + source.len(), + ) + }; + + assert_eq!(largest, 3); + assert_eq!(max_symbol, 3); + assert_eq!(&counts[..4], &[1, 0, 2, 3]); + assert!(counts[4..].iter().all(|&value| value == 0)); + } + + #[test] + fn empty_input_clears_requested_alphabet() { + let mut counts = [7u32; 8]; + let mut max_symbol = 7; + let mut workspace = [0u32; HIST_WKSP_SIZE_U32]; + + let largest = unsafe { + HIST_count_wksp( + counts.as_mut_ptr(), + &mut max_symbol, + std::ptr::null(), + 0, + workspace.as_mut_ptr().cast(), + HIST_WKSP_SIZE, + ) + }; + + assert_eq!(largest, 0); + assert_eq!(max_symbol, 0); + assert_eq!(counts, [0; 8]); + } + + #[test] + fn checked_count_rejects_symbols_outside_the_alphabet() { + let source = [1u8, 2, 7]; + let mut counts = [55u32; 8]; + let mut max_symbol = 3; + let mut workspace = [0u32; HIST_WKSP_SIZE_U32]; + + let result = unsafe { + HIST_count_wksp( + counts.as_mut_ptr(), + &mut max_symbol, + source.as_ptr().cast(), + source.len(), + workspace.as_mut_ptr().cast(), + HIST_WKSP_SIZE, + ) + }; + + assert_eq!(result, ERROR(ZstdErrorCode::MaxSymbolValueTooSmall)); + assert_eq!(max_symbol, 3); + assert_eq!(counts, [55; 8]); + } + + #[test] + fn workspace_contract_matches_the_c_api() { + let source = vec![42u8; 1500]; + let mut counts = [0u32; 256]; + let mut max_symbol = 255; + let mut workspace = vec![0u8; HIST_WKSP_SIZE + 1]; + + let misaligned = unsafe { + HIST_countFast_wksp( + counts.as_mut_ptr(), + &mut max_symbol, + source.as_ptr().cast(), + source.len(), + workspace.as_mut_ptr().add(1).cast(), + HIST_WKSP_SIZE, + ) + }; + assert_eq!(misaligned, ERROR(ZstdErrorCode::Generic)); + + let mut aligned_workspace = [0u32; HIST_WKSP_SIZE_U32]; + let too_small = unsafe { + HIST_countFast_wksp( + counts.as_mut_ptr(), + &mut max_symbol, + source.as_ptr().cast(), + source.len(), + aligned_workspace.as_mut_ptr().cast(), + HIST_WKSP_SIZE - 1, + ) + }; + assert_eq!(too_small, ERROR(ZstdErrorCode::WorkSpaceTooSmall)); + } + + #[test] + fn parallel_count_handles_stripes_and_tail() { + let source: Vec = (0..1537).map(|value| (value % 17) as u8).collect(); + let mut counts = [0u32; 256]; + let mut max_symbol = 255; + let mut workspace = [0u32; HIST_WKSP_SIZE_U32]; + + let largest = unsafe { + HIST_countFast_wksp( + counts.as_mut_ptr(), + &mut max_symbol, + source.as_ptr().cast(), + source.len(), + workspace.as_mut_ptr().cast(), + HIST_WKSP_SIZE, + ) + }; + + assert_eq!(largest, 91); + assert_eq!(max_symbol, 16); + assert_eq!(counts[..7], [91; 7]); + assert_eq!(counts[7..17], [90; 10]); + assert!(counts[17..].iter().all(|&value| value == 0)); + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 2d6004014..7990c0f2d 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -8,6 +8,7 @@ pub mod debug; pub mod entropy_common; pub mod errors; pub mod fse_decompress; +pub mod hist; pub mod mem; pub mod pool; pub mod threading;