feat(rust): port histogram counting primitives
Move byte histogram counting into Rust for FSE and Huffman compression callers. The implementation preserves the C workspace contract, checked alphabet path, fast path, empty-input behavior, and C ABI while the original C source becomes a header-only compatibility shim. Focused tests cover count accumulation, invalid alphabets, workspace failures, and striped large inputs. Higher-level compression remains C for now. Test Plan: - cargo fmt --check - cargo test --all-targets - cargo clippy --all-targets -- -D warnings - cargo build --release - make -C tests fuzzer - ./tests/fuzzer -i1 --no-big-tests - compile hist.c with -Werror and -Wredundant-decls Refs: rust/README.md
This commit is contained in:
+1
-189
@@ -1,191 +1,3 @@
|
||||
/* ******************************************************************
|
||||
* hist : Histogram functions
|
||||
* part of Finite State Entropy project
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
* - Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
****************************************************************** */
|
||||
|
||||
/* --- dependencies --- */
|
||||
#include "../common/mem.h" /* U32, BYTE, etc. */
|
||||
#include "../common/debug.h" /* assert, DEBUGLOG */
|
||||
#include "../common/error_private.h" /* ERROR */
|
||||
#include "hist.h"
|
||||
|
||||
|
||||
/* --- Error management --- */
|
||||
unsigned HIST_isError(size_t code) { return ERR_isError(code); }
|
||||
|
||||
/*-**************************************************************
|
||||
* Histogram functions
|
||||
****************************************************************/
|
||||
void HIST_add(unsigned* count, const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
const BYTE* const end = ip + srcSize;
|
||||
|
||||
while (ip<end) {
|
||||
count[*ip++]++;
|
||||
}
|
||||
}
|
||||
|
||||
unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
const BYTE* const end = ip + srcSize;
|
||||
unsigned maxSymbolValue = *maxSymbolValuePtr;
|
||||
unsigned largestCount=0;
|
||||
|
||||
ZSTD_memset(count, 0, (maxSymbolValue+1) * sizeof(*count));
|
||||
if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; }
|
||||
|
||||
while (ip<end) {
|
||||
assert(*ip <= maxSymbolValue);
|
||||
count[*ip++]++;
|
||||
}
|
||||
|
||||
while (!count[maxSymbolValue]) maxSymbolValue--;
|
||||
*maxSymbolValuePtr = maxSymbolValue;
|
||||
|
||||
{ U32 s;
|
||||
for (s=0; s<=maxSymbolValue; s++)
|
||||
if (count[s] > largestCount) largestCount = count[s];
|
||||
}
|
||||
|
||||
return largestCount;
|
||||
}
|
||||
|
||||
typedef enum { trustInput, checkMaxSymbolValue } HIST_checkInput_e;
|
||||
|
||||
/* HIST_count_parallel_wksp() :
|
||||
* store histogram into 4 intermediate tables, recombined at the end.
|
||||
* this design makes better use of OoO cpus,
|
||||
* and is noticeably faster when some values are heavily repeated.
|
||||
* But it needs some additional workspace for intermediate tables.
|
||||
* `workSpace` must be a U32 table of size >= HIST_WKSP_SIZE_U32.
|
||||
* @return : largest histogram frequency,
|
||||
* or an error code (notably when histogram's alphabet is larger than *maxSymbolValuePtr) */
|
||||
static size_t HIST_count_parallel_wksp(
|
||||
unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* source, size_t sourceSize,
|
||||
HIST_checkInput_e check,
|
||||
U32* const workSpace)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)source;
|
||||
const BYTE* const iend = ip+sourceSize;
|
||||
size_t const countSize = (*maxSymbolValuePtr + 1) * sizeof(*count);
|
||||
unsigned max=0;
|
||||
U32* const Counting1 = workSpace;
|
||||
U32* const Counting2 = Counting1 + 256;
|
||||
U32* const Counting3 = Counting2 + 256;
|
||||
U32* const Counting4 = Counting3 + 256;
|
||||
|
||||
/* safety checks */
|
||||
assert(*maxSymbolValuePtr <= 255);
|
||||
if (!sourceSize) {
|
||||
ZSTD_memset(count, 0, countSize);
|
||||
*maxSymbolValuePtr = 0;
|
||||
return 0;
|
||||
}
|
||||
ZSTD_memset(workSpace, 0, 4*256*sizeof(unsigned));
|
||||
|
||||
/* by stripes of 16 bytes */
|
||||
{ U32 cached = MEM_read32(ip); ip += 4;
|
||||
while (ip < iend-15) {
|
||||
U32 c = cached; cached = MEM_read32(ip); ip += 4;
|
||||
Counting1[(BYTE) c ]++;
|
||||
Counting2[(BYTE)(c>>8) ]++;
|
||||
Counting3[(BYTE)(c>>16)]++;
|
||||
Counting4[ c>>24 ]++;
|
||||
c = cached; cached = MEM_read32(ip); ip += 4;
|
||||
Counting1[(BYTE) c ]++;
|
||||
Counting2[(BYTE)(c>>8) ]++;
|
||||
Counting3[(BYTE)(c>>16)]++;
|
||||
Counting4[ c>>24 ]++;
|
||||
c = cached; cached = MEM_read32(ip); ip += 4;
|
||||
Counting1[(BYTE) c ]++;
|
||||
Counting2[(BYTE)(c>>8) ]++;
|
||||
Counting3[(BYTE)(c>>16)]++;
|
||||
Counting4[ c>>24 ]++;
|
||||
c = cached; cached = MEM_read32(ip); ip += 4;
|
||||
Counting1[(BYTE) c ]++;
|
||||
Counting2[(BYTE)(c>>8) ]++;
|
||||
Counting3[(BYTE)(c>>16)]++;
|
||||
Counting4[ c>>24 ]++;
|
||||
}
|
||||
ip-=4;
|
||||
}
|
||||
|
||||
/* finish last symbols */
|
||||
while (ip<iend) Counting1[*ip++]++;
|
||||
|
||||
{ U32 s;
|
||||
for (s=0; s<256; s++) {
|
||||
Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s];
|
||||
if (Counting1[s] > max) max = Counting1[s];
|
||||
} }
|
||||
|
||||
{ unsigned maxSymbolValue = 255;
|
||||
while (!Counting1[maxSymbolValue]) maxSymbolValue--;
|
||||
if (check && maxSymbolValue > *maxSymbolValuePtr) return ERROR(maxSymbolValue_tooSmall);
|
||||
*maxSymbolValuePtr = maxSymbolValue;
|
||||
ZSTD_memmove(count, Counting1, countSize); /* in case count & Counting1 are overlapping */
|
||||
}
|
||||
return (size_t)max;
|
||||
}
|
||||
|
||||
/* HIST_countFast_wksp() :
|
||||
* Same as HIST_countFast(), but using an externally provided scratch buffer.
|
||||
* `workSpace` is a writable buffer which must be 4-bytes aligned,
|
||||
* `workSpaceSize` must be >= HIST_WKSP_SIZE
|
||||
*/
|
||||
size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* source, size_t sourceSize,
|
||||
void* workSpace, size_t workSpaceSize)
|
||||
{
|
||||
if (sourceSize < 1500) /* heuristic threshold */
|
||||
return HIST_count_simple(count, maxSymbolValuePtr, source, sourceSize);
|
||||
if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
|
||||
if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall);
|
||||
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, trustInput, (U32*)workSpace);
|
||||
}
|
||||
|
||||
/* HIST_count_wksp() :
|
||||
* Same as HIST_count(), but using an externally provided scratch buffer.
|
||||
* `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */
|
||||
size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* source, size_t sourceSize,
|
||||
void* workSpace, size_t workSpaceSize)
|
||||
{
|
||||
if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
|
||||
if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall);
|
||||
if (*maxSymbolValuePtr < 255)
|
||||
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, checkMaxSymbolValue, (U32*)workSpace);
|
||||
*maxSymbolValuePtr = 255;
|
||||
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize);
|
||||
}
|
||||
|
||||
#ifndef ZSTD_NO_UNUSED_FUNCTIONS
|
||||
/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */
|
||||
size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* source, size_t sourceSize)
|
||||
{
|
||||
unsigned tmpCounters[HIST_WKSP_SIZE_U32];
|
||||
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters));
|
||||
}
|
||||
|
||||
size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
unsigned tmpCounters[HIST_WKSP_SIZE_U32];
|
||||
return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters));
|
||||
}
|
||||
#endif
|
||||
/* Implementation moved to rust/src/hist.rs. */
|
||||
|
||||
+8
-6
@@ -20,14 +20,16 @@ zstd ABI:
|
||||
- Entropy coding
|
||||
- `entropy_common` reads FSE normalized counts and Huffman statistics.
|
||||
- `fse_decompress` builds FSE decoding tables and decodes FSE streams.
|
||||
- Compression primitives
|
||||
- `hist` counts byte frequencies for FSE and Huffman compression.
|
||||
- Runtime support
|
||||
- `threading` provides platform pthread wrappers required by zstd headers.
|
||||
- `pool` implements the bounded worker pool used by multithreaded compression.
|
||||
|
||||
The remaining compression, general decompression, dictionary, legacy, and CLI
|
||||
translation units are still C. They must move before the rewrite is complete.
|
||||
Keeping that boundary explicit prevents a passing hybrid build from being
|
||||
mistaken for the final all-Rust result.
|
||||
The remaining block compression, general decompression, dictionary, legacy,
|
||||
and CLI translation units are still C. They must move before the rewrite is
|
||||
complete. Keeping that boundary explicit prevents a passing hybrid build from
|
||||
being mistaken for the final all-Rust result.
|
||||
|
||||
## Compatibility boundary
|
||||
|
||||
@@ -56,8 +58,8 @@ Then run original compatibility tests from the repository root, starting with
|
||||
the narrow target for the component being migrated. For example:
|
||||
|
||||
```sh
|
||||
make -C tests poolTests
|
||||
./tests/poolTests
|
||||
make -C tests fuzzer
|
||||
./tests/fuzzer -i1 --no-big-tests
|
||||
```
|
||||
|
||||
Broader `tests/Makefile` targets remain the authoritative integration gates as
|
||||
|
||||
@@ -0,0 +1,372 @@
|
||||
#![allow(non_snake_case)]
|
||||
|
||||
use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
|
||||
use std::ffi::c_void;
|
||||
use std::os::raw::c_uint;
|
||||
|
||||
pub const HIST_WKSP_SIZE_U32: usize = 1024;
|
||||
pub const HIST_WKSP_SIZE: usize = HIST_WKSP_SIZE_U32 * std::mem::size_of::<c_uint>();
|
||||
|
||||
#[no_mangle]
|
||||
pub extern "C" fn HIST_isError(code: usize) -> c_uint {
|
||||
ERR_isError(code) as c_uint
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn HIST_add(count: *mut c_uint, src: *const c_void, src_size: usize) {
|
||||
let mut ip = src.cast::<u8>();
|
||||
let iend = ip.wrapping_add(src_size);
|
||||
while ip < iend {
|
||||
let counter = count.add(*ip as usize);
|
||||
*counter = (*counter).wrapping_add(1);
|
||||
ip = ip.add(1);
|
||||
}
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn HIST_count_simple(
|
||||
count: *mut c_uint,
|
||||
max_symbol_value_ptr: *mut c_uint,
|
||||
src: *const c_void,
|
||||
src_size: usize,
|
||||
) -> c_uint {
|
||||
let mut max_symbol_value = *max_symbol_value_ptr;
|
||||
std::ptr::write_bytes(count, 0, max_symbol_value as usize + 1);
|
||||
|
||||
if src_size == 0 {
|
||||
*max_symbol_value_ptr = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
let mut ip = src.cast::<u8>();
|
||||
let iend = ip.add(src_size);
|
||||
while ip < iend {
|
||||
let counter = count.add(*ip as usize);
|
||||
*counter = (*counter).wrapping_add(1);
|
||||
ip = ip.add(1);
|
||||
}
|
||||
|
||||
while *count.add(max_symbol_value as usize) == 0 {
|
||||
max_symbol_value -= 1;
|
||||
}
|
||||
*max_symbol_value_ptr = max_symbol_value;
|
||||
|
||||
let mut largest_count = 0;
|
||||
for symbol in 0..=max_symbol_value as usize {
|
||||
largest_count = largest_count.max(*count.add(symbol));
|
||||
}
|
||||
largest_count
|
||||
}
|
||||
|
||||
unsafe fn HIST_count_parallel_wksp(
|
||||
count: *mut c_uint,
|
||||
max_symbol_value_ptr: *mut c_uint,
|
||||
source: *const c_void,
|
||||
source_size: usize,
|
||||
check_max_symbol_value: bool,
|
||||
workspace: *mut c_uint,
|
||||
) -> usize {
|
||||
let requested_max = *max_symbol_value_ptr as usize;
|
||||
let count_size = requested_max + 1;
|
||||
|
||||
if source_size == 0 {
|
||||
std::ptr::write_bytes(count, 0, count_size);
|
||||
*max_symbol_value_ptr = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::ptr::write_bytes(workspace, 0, HIST_WKSP_SIZE_U32);
|
||||
let counting1 = workspace;
|
||||
let counting2 = workspace.add(256);
|
||||
let counting3 = workspace.add(512);
|
||||
let counting4 = workspace.add(768);
|
||||
|
||||
let source = source.cast::<u8>();
|
||||
let mut offset = 0;
|
||||
|
||||
// Keep at least four bytes for the scalar tail, matching the C loop's
|
||||
// pipelined 16-byte stripes without its intentional short-input over-read.
|
||||
while source_size - offset >= 20 {
|
||||
for word in 0..4 {
|
||||
let bytes = source.add(offset + word * 4);
|
||||
let counters = [counting1, counting2, counting3, counting4];
|
||||
for (byte, counting) in counters.into_iter().enumerate() {
|
||||
let counter = counting.add(*bytes.add(byte) as usize);
|
||||
*counter = (*counter).wrapping_add(1);
|
||||
}
|
||||
}
|
||||
offset += 16;
|
||||
}
|
||||
|
||||
while offset < source_size {
|
||||
let counter = counting1.add(*source.add(offset) as usize);
|
||||
*counter = (*counter).wrapping_add(1);
|
||||
offset += 1;
|
||||
}
|
||||
|
||||
let mut largest_count = 0;
|
||||
for symbol in 0..256 {
|
||||
let total = (*counting1.add(symbol))
|
||||
.wrapping_add(*counting2.add(symbol))
|
||||
.wrapping_add(*counting3.add(symbol))
|
||||
.wrapping_add(*counting4.add(symbol));
|
||||
*counting1.add(symbol) = total;
|
||||
largest_count = largest_count.max(total);
|
||||
}
|
||||
|
||||
let mut max_symbol_value = 255usize;
|
||||
while *counting1.add(max_symbol_value) == 0 {
|
||||
max_symbol_value -= 1;
|
||||
}
|
||||
if check_max_symbol_value && max_symbol_value > requested_max {
|
||||
return ERROR(ZstdErrorCode::MaxSymbolValueTooSmall);
|
||||
}
|
||||
|
||||
*max_symbol_value_ptr = max_symbol_value as c_uint;
|
||||
std::ptr::copy(counting1, count, count_size);
|
||||
largest_count as usize
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn HIST_countFast_wksp(
|
||||
count: *mut c_uint,
|
||||
max_symbol_value_ptr: *mut c_uint,
|
||||
source: *const c_void,
|
||||
source_size: usize,
|
||||
workspace: *mut c_void,
|
||||
workspace_size: usize,
|
||||
) -> usize {
|
||||
if source_size < 1500 {
|
||||
return HIST_count_simple(count, max_symbol_value_ptr, source, source_size) as usize;
|
||||
}
|
||||
if workspace as usize & 3 != 0 {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
}
|
||||
if workspace_size < HIST_WKSP_SIZE {
|
||||
return ERROR(ZstdErrorCode::WorkSpaceTooSmall);
|
||||
}
|
||||
HIST_count_parallel_wksp(
|
||||
count,
|
||||
max_symbol_value_ptr,
|
||||
source,
|
||||
source_size,
|
||||
false,
|
||||
workspace.cast(),
|
||||
)
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn HIST_count_wksp(
|
||||
count: *mut c_uint,
|
||||
max_symbol_value_ptr: *mut c_uint,
|
||||
source: *const c_void,
|
||||
source_size: usize,
|
||||
workspace: *mut c_void,
|
||||
workspace_size: usize,
|
||||
) -> usize {
|
||||
if workspace as usize & 3 != 0 {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
}
|
||||
if workspace_size < HIST_WKSP_SIZE {
|
||||
return ERROR(ZstdErrorCode::WorkSpaceTooSmall);
|
||||
}
|
||||
if *max_symbol_value_ptr < 255 {
|
||||
return HIST_count_parallel_wksp(
|
||||
count,
|
||||
max_symbol_value_ptr,
|
||||
source,
|
||||
source_size,
|
||||
true,
|
||||
workspace.cast(),
|
||||
);
|
||||
}
|
||||
|
||||
*max_symbol_value_ptr = 255;
|
||||
HIST_countFast_wksp(
|
||||
count,
|
||||
max_symbol_value_ptr,
|
||||
source,
|
||||
source_size,
|
||||
workspace,
|
||||
workspace_size,
|
||||
)
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn HIST_countFast(
|
||||
count: *mut c_uint,
|
||||
max_symbol_value_ptr: *mut c_uint,
|
||||
source: *const c_void,
|
||||
source_size: usize,
|
||||
) -> usize {
|
||||
let mut workspace = [0u32; HIST_WKSP_SIZE_U32];
|
||||
HIST_countFast_wksp(
|
||||
count,
|
||||
max_symbol_value_ptr,
|
||||
source,
|
||||
source_size,
|
||||
workspace.as_mut_ptr().cast(),
|
||||
HIST_WKSP_SIZE,
|
||||
)
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn HIST_count(
|
||||
count: *mut c_uint,
|
||||
max_symbol_value_ptr: *mut c_uint,
|
||||
source: *const c_void,
|
||||
source_size: usize,
|
||||
) -> usize {
|
||||
let mut workspace = [0u32; HIST_WKSP_SIZE_U32];
|
||||
HIST_count_wksp(
|
||||
count,
|
||||
max_symbol_value_ptr,
|
||||
source,
|
||||
source_size,
|
||||
workspace.as_mut_ptr().cast(),
|
||||
HIST_WKSP_SIZE,
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn add_accumulates_counts() {
|
||||
let source = [0u8, 1, 1, 3, 3, 3];
|
||||
let mut counts = [0u32; 256];
|
||||
counts[1] = 4;
|
||||
|
||||
unsafe { HIST_add(counts.as_mut_ptr(), source.as_ptr().cast(), source.len()) };
|
||||
|
||||
assert_eq!(counts[0], 1);
|
||||
assert_eq!(counts[1], 6);
|
||||
assert_eq!(counts[2], 0);
|
||||
assert_eq!(counts[3], 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn simple_count_reports_largest_count_and_symbol() {
|
||||
let source = [0u8, 3, 3, 2, 3, 2];
|
||||
let mut counts = [99u32; 256];
|
||||
let mut max_symbol = 255;
|
||||
|
||||
let largest = unsafe {
|
||||
HIST_count_simple(
|
||||
counts.as_mut_ptr(),
|
||||
&mut max_symbol,
|
||||
source.as_ptr().cast(),
|
||||
source.len(),
|
||||
)
|
||||
};
|
||||
|
||||
assert_eq!(largest, 3);
|
||||
assert_eq!(max_symbol, 3);
|
||||
assert_eq!(&counts[..4], &[1, 0, 2, 3]);
|
||||
assert!(counts[4..].iter().all(|&value| value == 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_input_clears_requested_alphabet() {
|
||||
let mut counts = [7u32; 8];
|
||||
let mut max_symbol = 7;
|
||||
let mut workspace = [0u32; HIST_WKSP_SIZE_U32];
|
||||
|
||||
let largest = unsafe {
|
||||
HIST_count_wksp(
|
||||
counts.as_mut_ptr(),
|
||||
&mut max_symbol,
|
||||
std::ptr::null(),
|
||||
0,
|
||||
workspace.as_mut_ptr().cast(),
|
||||
HIST_WKSP_SIZE,
|
||||
)
|
||||
};
|
||||
|
||||
assert_eq!(largest, 0);
|
||||
assert_eq!(max_symbol, 0);
|
||||
assert_eq!(counts, [0; 8]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checked_count_rejects_symbols_outside_the_alphabet() {
|
||||
let source = [1u8, 2, 7];
|
||||
let mut counts = [55u32; 8];
|
||||
let mut max_symbol = 3;
|
||||
let mut workspace = [0u32; HIST_WKSP_SIZE_U32];
|
||||
|
||||
let result = unsafe {
|
||||
HIST_count_wksp(
|
||||
counts.as_mut_ptr(),
|
||||
&mut max_symbol,
|
||||
source.as_ptr().cast(),
|
||||
source.len(),
|
||||
workspace.as_mut_ptr().cast(),
|
||||
HIST_WKSP_SIZE,
|
||||
)
|
||||
};
|
||||
|
||||
assert_eq!(result, ERROR(ZstdErrorCode::MaxSymbolValueTooSmall));
|
||||
assert_eq!(max_symbol, 3);
|
||||
assert_eq!(counts, [55; 8]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workspace_contract_matches_the_c_api() {
|
||||
let source = vec![42u8; 1500];
|
||||
let mut counts = [0u32; 256];
|
||||
let mut max_symbol = 255;
|
||||
let mut workspace = vec![0u8; HIST_WKSP_SIZE + 1];
|
||||
|
||||
let misaligned = unsafe {
|
||||
HIST_countFast_wksp(
|
||||
counts.as_mut_ptr(),
|
||||
&mut max_symbol,
|
||||
source.as_ptr().cast(),
|
||||
source.len(),
|
||||
workspace.as_mut_ptr().add(1).cast(),
|
||||
HIST_WKSP_SIZE,
|
||||
)
|
||||
};
|
||||
assert_eq!(misaligned, ERROR(ZstdErrorCode::Generic));
|
||||
|
||||
let mut aligned_workspace = [0u32; HIST_WKSP_SIZE_U32];
|
||||
let too_small = unsafe {
|
||||
HIST_countFast_wksp(
|
||||
counts.as_mut_ptr(),
|
||||
&mut max_symbol,
|
||||
source.as_ptr().cast(),
|
||||
source.len(),
|
||||
aligned_workspace.as_mut_ptr().cast(),
|
||||
HIST_WKSP_SIZE - 1,
|
||||
)
|
||||
};
|
||||
assert_eq!(too_small, ERROR(ZstdErrorCode::WorkSpaceTooSmall));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parallel_count_handles_stripes_and_tail() {
|
||||
let source: Vec<u8> = (0..1537).map(|value| (value % 17) as u8).collect();
|
||||
let mut counts = [0u32; 256];
|
||||
let mut max_symbol = 255;
|
||||
let mut workspace = [0u32; HIST_WKSP_SIZE_U32];
|
||||
|
||||
let largest = unsafe {
|
||||
HIST_countFast_wksp(
|
||||
counts.as_mut_ptr(),
|
||||
&mut max_symbol,
|
||||
source.as_ptr().cast(),
|
||||
source.len(),
|
||||
workspace.as_mut_ptr().cast(),
|
||||
HIST_WKSP_SIZE,
|
||||
)
|
||||
};
|
||||
|
||||
assert_eq!(largest, 91);
|
||||
assert_eq!(max_symbol, 16);
|
||||
assert_eq!(counts[..7], [91; 7]);
|
||||
assert_eq!(counts[7..17], [90; 10]);
|
||||
assert!(counts[17..].iter().all(|&value| value == 0));
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ pub mod debug;
|
||||
pub mod entropy_common;
|
||||
pub mod errors;
|
||||
pub mod fse_decompress;
|
||||
pub mod hist;
|
||||
pub mod mem;
|
||||
pub mod pool;
|
||||
pub mod threading;
|
||||
|
||||
Reference in New Issue
Block a user