feat(rust): port compression block pre-splitting
Move ZSTD_splitBlock into Rust while retaining its caller-owned workspace contract. The implementation preserves the C fingerprint sampling heuristic and calls the migrated histogram primitive without adding a hot-path allocation. Reference-vector and randomized differential tests compare every split level with the original C translation unit. The C source remains as a declaration shim so existing source lists resolve the Rust ABI during the migration. Test Plan: - cargo fmt, cargo clippy, cargo clippy --benches, cargo clippy --tests - cargo test --all-targets and cargo build --release - cargo test/build --target i686-unknown-linux-gnu - 264-case original-C/Rust differential harness - C fuzzer, zstreamtest, and fuzzer32 smoke runs Refs: rust/README.md
This commit is contained in:
@@ -1,238 +1,2 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#include "../common/compiler.h" /* ZSTD_ALIGNOF */
|
||||
#include "../common/mem.h" /* S64 */
|
||||
#include "../common/zstd_deps.h" /* ZSTD_memset */
|
||||
#include "../common/zstd_internal.h" /* ZSTD_STATIC_ASSERT */
|
||||
#include "hist.h" /* HIST_add */
|
||||
/* Implementation moved to rust/src/zstd_presplit.rs. */
|
||||
#include "zstd_preSplit.h"
|
||||
|
||||
|
||||
#define BLOCKSIZE_MIN 3500
|
||||
#define THRESHOLD_PENALTY_RATE 16
|
||||
#define THRESHOLD_BASE (THRESHOLD_PENALTY_RATE - 2)
|
||||
#define THRESHOLD_PENALTY 3
|
||||
|
||||
#define HASHLENGTH 2
|
||||
#define HASHLOG_MAX 10
|
||||
#define HASHTABLESIZE (1 << HASHLOG_MAX)
|
||||
#define HASHMASK (HASHTABLESIZE - 1)
|
||||
#define KNUTH 0x9e3779b9
|
||||
|
||||
/* for hashLog > 8, hash 2 bytes.
|
||||
* for hashLog == 8, just take the byte, no hashing.
|
||||
* The speed of this method relies on compile-time constant propagation */
|
||||
FORCE_INLINE_TEMPLATE unsigned hash2(const void *p, unsigned hashLog)
|
||||
{
|
||||
assert(hashLog >= 8);
|
||||
if (hashLog == 8) return (U32)((const BYTE*)p)[0];
|
||||
assert(hashLog <= HASHLOG_MAX);
|
||||
return (U32)(MEM_read16(p)) * KNUTH >> (32 - hashLog);
|
||||
}
|
||||
|
||||
|
||||
typedef struct {
|
||||
unsigned events[HASHTABLESIZE];
|
||||
size_t nbEvents;
|
||||
} Fingerprint;
|
||||
typedef struct {
|
||||
Fingerprint pastEvents;
|
||||
Fingerprint newEvents;
|
||||
} FPStats;
|
||||
|
||||
static void initStats(FPStats* fpstats)
|
||||
{
|
||||
ZSTD_memset(fpstats, 0, sizeof(FPStats));
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE void
|
||||
addEvents_generic(Fingerprint* fp, const void* src, size_t srcSize, size_t samplingRate, unsigned hashLog)
|
||||
{
|
||||
const char* p = (const char*)src;
|
||||
size_t limit = srcSize - HASHLENGTH + 1;
|
||||
size_t n;
|
||||
assert(srcSize >= HASHLENGTH);
|
||||
for (n = 0; n < limit; n+=samplingRate) {
|
||||
fp->events[hash2(p+n, hashLog)]++;
|
||||
}
|
||||
fp->nbEvents += limit/samplingRate;
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE void
|
||||
recordFingerprint_generic(Fingerprint* fp, const void* src, size_t srcSize, size_t samplingRate, unsigned hashLog)
|
||||
{
|
||||
ZSTD_memset(fp, 0, sizeof(unsigned) * ((size_t)1 << hashLog));
|
||||
fp->nbEvents = 0;
|
||||
addEvents_generic(fp, src, srcSize, samplingRate, hashLog);
|
||||
}
|
||||
|
||||
typedef void (*RecordEvents_f)(Fingerprint* fp, const void* src, size_t srcSize);
|
||||
|
||||
#define FP_RECORD(_rate) ZSTD_recordFingerprint_##_rate
|
||||
|
||||
#define ZSTD_GEN_RECORD_FINGERPRINT(_rate, _hSize) \
|
||||
static void FP_RECORD(_rate)(Fingerprint* fp, const void* src, size_t srcSize) \
|
||||
{ \
|
||||
recordFingerprint_generic(fp, src, srcSize, _rate, _hSize); \
|
||||
}
|
||||
|
||||
ZSTD_GEN_RECORD_FINGERPRINT(1, 10)
|
||||
ZSTD_GEN_RECORD_FINGERPRINT(5, 10)
|
||||
ZSTD_GEN_RECORD_FINGERPRINT(11, 9)
|
||||
ZSTD_GEN_RECORD_FINGERPRINT(43, 8)
|
||||
|
||||
|
||||
static U64 abs64(S64 s64) { return (U64)((s64 < 0) ? -s64 : s64); }
|
||||
|
||||
static U64 fpDistance(const Fingerprint* fp1, const Fingerprint* fp2, unsigned hashLog)
|
||||
{
|
||||
U64 distance = 0;
|
||||
size_t n;
|
||||
assert(hashLog <= HASHLOG_MAX);
|
||||
for (n = 0; n < ((size_t)1 << hashLog); n++) {
|
||||
distance +=
|
||||
abs64((S64)fp1->events[n] * (S64)fp2->nbEvents - (S64)fp2->events[n] * (S64)fp1->nbEvents);
|
||||
}
|
||||
return distance;
|
||||
}
|
||||
|
||||
/* Compare newEvents with pastEvents
|
||||
* return 1 when considered "too different"
|
||||
*/
|
||||
static int compareFingerprints(const Fingerprint* ref,
|
||||
const Fingerprint* newfp,
|
||||
int penalty,
|
||||
unsigned hashLog)
|
||||
{
|
||||
assert(ref->nbEvents > 0);
|
||||
assert(newfp->nbEvents > 0);
|
||||
{ U64 p50 = (U64)ref->nbEvents * (U64)newfp->nbEvents;
|
||||
U64 deviation = fpDistance(ref, newfp, hashLog);
|
||||
U64 threshold = p50 * (U64)(THRESHOLD_BASE + penalty) / THRESHOLD_PENALTY_RATE;
|
||||
return deviation >= threshold;
|
||||
}
|
||||
}
|
||||
|
||||
static void mergeEvents(Fingerprint* acc, const Fingerprint* newfp)
|
||||
{
|
||||
size_t n;
|
||||
for (n = 0; n < HASHTABLESIZE; n++) {
|
||||
acc->events[n] += newfp->events[n];
|
||||
}
|
||||
acc->nbEvents += newfp->nbEvents;
|
||||
}
|
||||
|
||||
static void flushEvents(FPStats* fpstats)
|
||||
{
|
||||
size_t n;
|
||||
for (n = 0; n < HASHTABLESIZE; n++) {
|
||||
fpstats->pastEvents.events[n] = fpstats->newEvents.events[n];
|
||||
}
|
||||
fpstats->pastEvents.nbEvents = fpstats->newEvents.nbEvents;
|
||||
ZSTD_memset(&fpstats->newEvents, 0, sizeof(fpstats->newEvents));
|
||||
}
|
||||
|
||||
static void removeEvents(Fingerprint* acc, const Fingerprint* slice)
|
||||
{
|
||||
size_t n;
|
||||
for (n = 0; n < HASHTABLESIZE; n++) {
|
||||
assert(acc->events[n] >= slice->events[n]);
|
||||
acc->events[n] -= slice->events[n];
|
||||
}
|
||||
acc->nbEvents -= slice->nbEvents;
|
||||
}
|
||||
|
||||
#define CHUNKSIZE (8 << 10)
|
||||
static size_t ZSTD_splitBlock_byChunks(const void* blockStart, size_t blockSize,
|
||||
int level,
|
||||
void* workspace, size_t wkspSize)
|
||||
{
|
||||
static const RecordEvents_f records_fs[] = {
|
||||
FP_RECORD(43), FP_RECORD(11), FP_RECORD(5), FP_RECORD(1)
|
||||
};
|
||||
static const unsigned hashParams[] = { 8, 9, 10, 10 };
|
||||
const RecordEvents_f record_f = (assert(0<=level && level<=3), records_fs[level]);
|
||||
FPStats* const fpstats = (FPStats*)workspace;
|
||||
const char* p = (const char*)blockStart;
|
||||
int penalty = THRESHOLD_PENALTY;
|
||||
size_t pos = 0;
|
||||
assert(blockSize == (128 << 10));
|
||||
assert(workspace != NULL);
|
||||
assert((size_t)workspace % ZSTD_ALIGNOF(FPStats) == 0);
|
||||
ZSTD_STATIC_ASSERT(ZSTD_SLIPBLOCK_WORKSPACESIZE >= sizeof(FPStats));
|
||||
assert(wkspSize >= sizeof(FPStats)); (void)wkspSize;
|
||||
|
||||
initStats(fpstats);
|
||||
record_f(&fpstats->pastEvents, p, CHUNKSIZE);
|
||||
for (pos = CHUNKSIZE; pos <= blockSize - CHUNKSIZE; pos += CHUNKSIZE) {
|
||||
record_f(&fpstats->newEvents, p + pos, CHUNKSIZE);
|
||||
if (compareFingerprints(&fpstats->pastEvents, &fpstats->newEvents, penalty, hashParams[level])) {
|
||||
return pos;
|
||||
} else {
|
||||
mergeEvents(&fpstats->pastEvents, &fpstats->newEvents);
|
||||
if (penalty > 0) penalty--;
|
||||
}
|
||||
}
|
||||
assert(pos == blockSize);
|
||||
return blockSize;
|
||||
(void)flushEvents; (void)removeEvents;
|
||||
}
|
||||
|
||||
/* ZSTD_splitBlock_fromBorders(): very fast strategy :
|
||||
* compare fingerprint from beginning and end of the block,
|
||||
* derive from their difference if it's preferable to split in the middle,
|
||||
* repeat the process a second time, for finer grained decision.
|
||||
* 3 times did not brought improvements, so I stopped at 2.
|
||||
* Benefits are good enough for a cheap heuristic.
|
||||
* More accurate splitting saves more, but speed impact is also more perceptible.
|
||||
* For better accuracy, use more elaborate variant *_byChunks.
|
||||
*/
|
||||
static size_t ZSTD_splitBlock_fromBorders(const void* blockStart, size_t blockSize,
|
||||
void* workspace, size_t wkspSize)
|
||||
{
|
||||
#define SEGMENT_SIZE 512
|
||||
FPStats* const fpstats = (FPStats*)workspace;
|
||||
Fingerprint* middleEvents = (Fingerprint*)(void*)((char*)workspace + 512 * sizeof(unsigned));
|
||||
assert(blockSize == (128 << 10));
|
||||
assert(workspace != NULL);
|
||||
assert((size_t)workspace % ZSTD_ALIGNOF(FPStats) == 0);
|
||||
ZSTD_STATIC_ASSERT(ZSTD_SLIPBLOCK_WORKSPACESIZE >= sizeof(FPStats));
|
||||
assert(wkspSize >= sizeof(FPStats)); (void)wkspSize;
|
||||
|
||||
initStats(fpstats);
|
||||
HIST_add(fpstats->pastEvents.events, blockStart, SEGMENT_SIZE);
|
||||
HIST_add(fpstats->newEvents.events, (const char*)blockStart + blockSize - SEGMENT_SIZE, SEGMENT_SIZE);
|
||||
fpstats->pastEvents.nbEvents = fpstats->newEvents.nbEvents = SEGMENT_SIZE;
|
||||
if (!compareFingerprints(&fpstats->pastEvents, &fpstats->newEvents, 0, 8))
|
||||
return blockSize;
|
||||
|
||||
HIST_add(middleEvents->events, (const char*)blockStart + blockSize/2 - SEGMENT_SIZE/2, SEGMENT_SIZE);
|
||||
middleEvents->nbEvents = SEGMENT_SIZE;
|
||||
{ U64 const distFromBegin = fpDistance(&fpstats->pastEvents, middleEvents, 8);
|
||||
U64 const distFromEnd = fpDistance(&fpstats->newEvents, middleEvents, 8);
|
||||
U64 const minDistance = SEGMENT_SIZE * SEGMENT_SIZE / 3;
|
||||
if (abs64((S64)distFromBegin - (S64)distFromEnd) < minDistance)
|
||||
return 64 KB;
|
||||
return (distFromBegin > distFromEnd) ? 32 KB : 96 KB;
|
||||
}
|
||||
}
|
||||
|
||||
size_t ZSTD_splitBlock(const void* blockStart, size_t blockSize,
|
||||
int level,
|
||||
void* workspace, size_t wkspSize)
|
||||
{
|
||||
DEBUGLOG(6, "ZSTD_splitBlock (level=%i)", level);
|
||||
assert(0<=level && level<=4);
|
||||
if (level == 0)
|
||||
return ZSTD_splitBlock_fromBorders(blockStart, blockSize, workspace, wkspSize);
|
||||
/* level >= 1*/
|
||||
return ZSTD_splitBlock_byChunks(blockStart, blockSize, level-1, workspace, wkspSize);
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ zstd ABI:
|
||||
Huffman streams.
|
||||
- Compression primitives
|
||||
- `hist` counts byte frequencies for FSE and Huffman compression.
|
||||
- `zstd_presplit` chooses split points for full compression blocks.
|
||||
- Runtime support
|
||||
- `threading` provides platform pthread wrappers required by zstd headers.
|
||||
- `pool` implements the bounded worker pool used by multithreaded compression.
|
||||
|
||||
@@ -17,3 +17,4 @@ pub mod threading;
|
||||
pub mod xxhash;
|
||||
pub mod zstd_common;
|
||||
pub mod zstd_ddict;
|
||||
pub mod zstd_presplit;
|
||||
|
||||
@@ -0,0 +1,376 @@
|
||||
#![allow(non_snake_case)]
|
||||
|
||||
//! Compression block pre-splitting.
|
||||
//!
|
||||
//! This is the small distribution heuristic used by the compressor to decide
|
||||
//! whether a full 128 KiB block should be split before match finding. It uses
|
||||
//! caller-provided workspace exactly like the C implementation so it does not
|
||||
//! add an allocation on the compression hot path.
|
||||
|
||||
use crate::hist::HIST_add;
|
||||
use crate::mem::MEM_read16;
|
||||
use std::ffi::c_void;
|
||||
use std::mem::size_of;
|
||||
use std::os::raw::c_int;
|
||||
use std::ptr;
|
||||
|
||||
const BLOCK_SIZE: usize = 128 << 10;
|
||||
const BLOCK_SIZE_MIN: usize = 3_500;
|
||||
const THRESHOLD_PENALTY_RATE: u64 = 16;
|
||||
const THRESHOLD_BASE: u64 = THRESHOLD_PENALTY_RATE - 2;
|
||||
const THRESHOLD_PENALTY: u64 = 3;
|
||||
|
||||
const HASH_LENGTH: usize = 2;
|
||||
const HASH_LOG_MAX: u32 = 10;
|
||||
const HASH_TABLE_SIZE: usize = 1 << HASH_LOG_MAX;
|
||||
const KNUTH: u32 = 0x9e37_79b9;
|
||||
const CHUNK_SIZE: usize = 8 << 10;
|
||||
const SEGMENT_SIZE: usize = 512;
|
||||
const WORKSPACE_SIZE: usize = 8_208;
|
||||
|
||||
#[repr(C)]
|
||||
struct Fingerprint {
|
||||
events: [u32; HASH_TABLE_SIZE],
|
||||
nb_events: usize,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
struct FpStats {
|
||||
past_events: Fingerprint,
|
||||
new_events: Fingerprint,
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn fingerprint_events(fp: *mut Fingerprint) -> *mut u32 {
|
||||
unsafe { ptr::addr_of_mut!((*fp).events).cast() }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn fingerprint_nb_events(fp: *const Fingerprint) -> usize {
|
||||
unsafe { ptr::addr_of!((*fp).nb_events).read() }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn set_fingerprint_nb_events(fp: *mut Fingerprint, value: usize) {
|
||||
unsafe { ptr::addr_of_mut!((*fp).nb_events).write(value) }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn hash2(source: *const u8, hash_log: u32) -> usize {
|
||||
debug_assert!((8..=HASH_LOG_MAX).contains(&hash_log));
|
||||
if hash_log == 8 {
|
||||
unsafe { *source as usize }
|
||||
} else {
|
||||
let value = unsafe { MEM_read16(source.cast()) as u32 };
|
||||
(value.wrapping_mul(KNUTH) >> (32 - hash_log)) as usize
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn init_stats(stats: *mut FpStats) {
|
||||
unsafe { ptr::write_bytes(stats.cast::<u8>(), 0, size_of::<FpStats>()) }
|
||||
}
|
||||
|
||||
unsafe fn add_events(
|
||||
fp: *mut Fingerprint,
|
||||
source: *const u8,
|
||||
source_size: usize,
|
||||
sampling_rate: usize,
|
||||
hash_log: u32,
|
||||
) {
|
||||
debug_assert!(source_size >= HASH_LENGTH);
|
||||
let limit = source_size - HASH_LENGTH + 1;
|
||||
let events = unsafe { fingerprint_events(fp) };
|
||||
let mut index = 0;
|
||||
while index < limit {
|
||||
let event = unsafe { events.add(hash2(source.add(index), hash_log)) };
|
||||
unsafe { event.write(event.read().wrapping_add(1)) };
|
||||
index += sampling_rate;
|
||||
}
|
||||
let nb_events = unsafe { fingerprint_nb_events(fp) };
|
||||
unsafe { set_fingerprint_nb_events(fp, nb_events.wrapping_add(limit / sampling_rate)) };
|
||||
}
|
||||
|
||||
unsafe fn record_fingerprint(
|
||||
fp: *mut Fingerprint,
|
||||
source: *const u8,
|
||||
source_size: usize,
|
||||
sampling_rate: usize,
|
||||
hash_log: u32,
|
||||
) {
|
||||
unsafe {
|
||||
ptr::write_bytes(fingerprint_events(fp), 0, 1usize << hash_log);
|
||||
set_fingerprint_nb_events(fp, 0);
|
||||
add_events(fp, source, source_size, sampling_rate, hash_log);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn abs64(value: i64) -> u64 {
|
||||
if value < 0 {
|
||||
value.wrapping_neg() as u64
|
||||
} else {
|
||||
value as u64
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn fp_distance(fp1: *const Fingerprint, fp2: *const Fingerprint, hash_log: u32) -> u64 {
|
||||
debug_assert!(hash_log <= HASH_LOG_MAX);
|
||||
let fp1_events = unsafe { fingerprint_events(fp1.cast_mut()) };
|
||||
let fp2_events = unsafe { fingerprint_events(fp2.cast_mut()) };
|
||||
let fp1_nb_events = unsafe { fingerprint_nb_events(fp1) } as i64;
|
||||
let fp2_nb_events = unsafe { fingerprint_nb_events(fp2) } as i64;
|
||||
let mut distance = 0u64;
|
||||
for index in 0..(1usize << hash_log) {
|
||||
let lhs = unsafe { fp1_events.add(index).read() as i64 } * fp2_nb_events;
|
||||
let rhs = unsafe { fp2_events.add(index).read() as i64 } * fp1_nb_events;
|
||||
distance = distance.wrapping_add(abs64(lhs.wrapping_sub(rhs)));
|
||||
}
|
||||
distance
|
||||
}
|
||||
|
||||
unsafe fn compare_fingerprints(
|
||||
reference: *const Fingerprint,
|
||||
new_fingerprint: *const Fingerprint,
|
||||
penalty: u64,
|
||||
hash_log: u32,
|
||||
) -> bool {
|
||||
let reference_events = unsafe { fingerprint_nb_events(reference) } as u64;
|
||||
let new_events = unsafe { fingerprint_nb_events(new_fingerprint) } as u64;
|
||||
debug_assert_ne!(reference_events, 0);
|
||||
debug_assert_ne!(new_events, 0);
|
||||
|
||||
let p50 = reference_events * new_events;
|
||||
let deviation = unsafe { fp_distance(reference, new_fingerprint, hash_log) };
|
||||
let threshold = p50 * (THRESHOLD_BASE + penalty) / THRESHOLD_PENALTY_RATE;
|
||||
deviation >= threshold
|
||||
}
|
||||
|
||||
unsafe fn merge_events(accumulator: *mut Fingerprint, new_fingerprint: *const Fingerprint) {
|
||||
let accumulator_events = unsafe { fingerprint_events(accumulator) };
|
||||
let new_events = unsafe { fingerprint_events(new_fingerprint.cast_mut()) };
|
||||
for index in 0..HASH_TABLE_SIZE {
|
||||
let value = unsafe {
|
||||
accumulator_events
|
||||
.add(index)
|
||||
.read()
|
||||
.wrapping_add(new_events.add(index).read())
|
||||
};
|
||||
unsafe { accumulator_events.add(index).write(value) };
|
||||
}
|
||||
let nb_events = unsafe { fingerprint_nb_events(accumulator) }
|
||||
.wrapping_add(unsafe { fingerprint_nb_events(new_fingerprint) });
|
||||
unsafe { set_fingerprint_nb_events(accumulator, nb_events) };
|
||||
}
|
||||
|
||||
unsafe fn split_block_by_chunks(
|
||||
block_start: *const u8,
|
||||
block_size: usize,
|
||||
level: usize,
|
||||
workspace: *mut c_void,
|
||||
workspace_size: usize,
|
||||
) -> usize {
|
||||
const RECORD_PARAMS: [(usize, u32); 4] = [(43, 8), (11, 9), (5, 10), (1, 10)];
|
||||
|
||||
debug_assert!(level <= 3);
|
||||
debug_assert_eq!(block_size, BLOCK_SIZE);
|
||||
debug_assert!(!workspace.is_null());
|
||||
debug_assert_eq!(workspace as usize % std::mem::align_of::<FpStats>(), 0);
|
||||
debug_assert!(workspace_size >= size_of::<FpStats>());
|
||||
debug_assert!(workspace_size >= WORKSPACE_SIZE);
|
||||
|
||||
let (sampling_rate, hash_log) = RECORD_PARAMS[level];
|
||||
let stats = workspace.cast::<FpStats>();
|
||||
let past_events = unsafe { ptr::addr_of_mut!((*stats).past_events) };
|
||||
let new_events = unsafe { ptr::addr_of_mut!((*stats).new_events) };
|
||||
unsafe {
|
||||
init_stats(stats);
|
||||
record_fingerprint(
|
||||
past_events,
|
||||
block_start,
|
||||
CHUNK_SIZE,
|
||||
sampling_rate,
|
||||
hash_log,
|
||||
);
|
||||
}
|
||||
|
||||
let mut penalty = THRESHOLD_PENALTY;
|
||||
let mut position = CHUNK_SIZE;
|
||||
while position <= block_size - CHUNK_SIZE {
|
||||
unsafe {
|
||||
record_fingerprint(
|
||||
new_events,
|
||||
block_start.add(position),
|
||||
CHUNK_SIZE,
|
||||
sampling_rate,
|
||||
hash_log,
|
||||
);
|
||||
}
|
||||
if unsafe { compare_fingerprints(past_events, new_events, penalty, hash_log) } {
|
||||
return position;
|
||||
}
|
||||
unsafe { merge_events(past_events, new_events) };
|
||||
penalty = penalty.saturating_sub(1);
|
||||
position += CHUNK_SIZE;
|
||||
}
|
||||
debug_assert_eq!(position, block_size);
|
||||
block_size
|
||||
}
|
||||
|
||||
unsafe fn split_block_from_borders(
|
||||
block_start: *const u8,
|
||||
block_size: usize,
|
||||
workspace: *mut c_void,
|
||||
workspace_size: usize,
|
||||
) -> usize {
|
||||
debug_assert_eq!(block_size, BLOCK_SIZE);
|
||||
debug_assert!(!workspace.is_null());
|
||||
debug_assert_eq!(workspace as usize % std::mem::align_of::<FpStats>(), 0);
|
||||
debug_assert!(workspace_size >= size_of::<FpStats>());
|
||||
debug_assert!(workspace_size >= WORKSPACE_SIZE);
|
||||
|
||||
let stats = workspace.cast::<FpStats>();
|
||||
let past_events = unsafe { ptr::addr_of_mut!((*stats).past_events) };
|
||||
let new_events = unsafe { ptr::addr_of_mut!((*stats).new_events) };
|
||||
let middle_events = unsafe { workspace.cast::<u8>().add(SEGMENT_SIZE * size_of::<u32>()) }
|
||||
.cast::<Fingerprint>();
|
||||
|
||||
unsafe {
|
||||
init_stats(stats);
|
||||
HIST_add(
|
||||
fingerprint_events(past_events),
|
||||
block_start.cast(),
|
||||
SEGMENT_SIZE,
|
||||
);
|
||||
HIST_add(
|
||||
fingerprint_events(new_events),
|
||||
block_start.add(block_size - SEGMENT_SIZE).cast(),
|
||||
SEGMENT_SIZE,
|
||||
);
|
||||
set_fingerprint_nb_events(past_events, SEGMENT_SIZE);
|
||||
set_fingerprint_nb_events(new_events, SEGMENT_SIZE);
|
||||
}
|
||||
if !unsafe { compare_fingerprints(past_events, new_events, 0, 8) } {
|
||||
return block_size;
|
||||
}
|
||||
|
||||
unsafe {
|
||||
HIST_add(
|
||||
fingerprint_events(middle_events),
|
||||
block_start.add(block_size / 2 - SEGMENT_SIZE / 2).cast(),
|
||||
SEGMENT_SIZE,
|
||||
);
|
||||
set_fingerprint_nb_events(middle_events, SEGMENT_SIZE);
|
||||
}
|
||||
let distance_from_begin = unsafe { fp_distance(past_events, middle_events, 8) };
|
||||
let distance_from_end = unsafe { fp_distance(new_events, middle_events, 8) };
|
||||
let minimum_distance = (SEGMENT_SIZE * SEGMENT_SIZE / 3) as u64;
|
||||
if abs64((distance_from_begin as i64).wrapping_sub(distance_from_end as i64)) < minimum_distance
|
||||
{
|
||||
64 << 10
|
||||
} else if distance_from_begin > distance_from_end {
|
||||
32 << 10
|
||||
} else {
|
||||
96 << 10
|
||||
}
|
||||
}
|
||||
|
||||
/// Chooses a split point for a full-size compression block.
|
||||
///
|
||||
/// # Safety
|
||||
/// `block_start` must reference exactly one readable 128 KiB block. `workspace`
|
||||
/// must be aligned for `FpStats` and provide at least 8,208 bytes.
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_splitBlock(
|
||||
block_start: *const c_void,
|
||||
block_size: usize,
|
||||
level: c_int,
|
||||
workspace: *mut c_void,
|
||||
workspace_size: usize,
|
||||
) -> usize {
|
||||
debug_assert!((0..=4).contains(&level));
|
||||
debug_assert!(block_size >= BLOCK_SIZE_MIN);
|
||||
if level == 0 {
|
||||
unsafe {
|
||||
split_block_from_borders(block_start.cast(), block_size, workspace, workspace_size)
|
||||
}
|
||||
} else {
|
||||
unsafe {
|
||||
split_block_by_chunks(
|
||||
block_start.cast(),
|
||||
block_size,
|
||||
(level - 1) as usize,
|
||||
workspace,
|
||||
workspace_size,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn split(input: &[u8], level: c_int) -> usize {
|
||||
let mut workspace = vec![0usize; WORKSPACE_SIZE.div_ceil(size_of::<usize>())];
|
||||
unsafe {
|
||||
ZSTD_splitBlock(
|
||||
input.as_ptr().cast(),
|
||||
input.len(),
|
||||
level,
|
||||
workspace.as_mut_ptr().cast(),
|
||||
workspace.len() * size_of::<usize>(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_all_levels(input: &[u8], expected: [usize; 5]) {
|
||||
for (level, expected) in expected.into_iter().enumerate() {
|
||||
assert_eq!(split(input, level as c_int), expected, "level {level}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workspace_layout_fits_the_public_contract() {
|
||||
assert!(size_of::<FpStats>() <= WORKSPACE_SIZE);
|
||||
assert_eq!(
|
||||
size_of::<Fingerprint>(),
|
||||
HASH_TABLE_SIZE * size_of::<u32>() + size_of::<usize>()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_points_match_c_reference_vectors() {
|
||||
let mut input = vec![0u8; BLOCK_SIZE];
|
||||
assert_all_levels(&input, [BLOCK_SIZE; 5]);
|
||||
|
||||
input[..BLOCK_SIZE / 2].fill(0);
|
||||
input[BLOCK_SIZE / 2..].fill(1);
|
||||
assert_all_levels(&input, [64 << 10; 5]);
|
||||
|
||||
input[..BLOCK_SIZE / 4].fill(0);
|
||||
input[BLOCK_SIZE / 4..].fill(1);
|
||||
assert_all_levels(&input, [32 << 10; 5]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sampled_levels_match_c_reference_vectors() {
|
||||
let mut input = vec![0u8; BLOCK_SIZE];
|
||||
let mut seed = 0x1234_5678u32;
|
||||
for byte in &mut input {
|
||||
seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
|
||||
*byte = (seed >> 24) as u8;
|
||||
}
|
||||
assert_all_levels(
|
||||
&input,
|
||||
[BLOCK_SIZE, CHUNK_SIZE, BLOCK_SIZE, BLOCK_SIZE, BLOCK_SIZE],
|
||||
);
|
||||
|
||||
for (index, byte) in input.iter_mut().enumerate() {
|
||||
*byte = ((index / CHUNK_SIZE) & 1) as u8;
|
||||
}
|
||||
assert_all_levels(
|
||||
&input,
|
||||
[64 << 10, CHUNK_SIZE, CHUNK_SIZE, CHUNK_SIZE, CHUNK_SIZE],
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user