feat(rust): port compression literal blocks
Move raw, RLE, and Huffman literal-section encoding into the Rust compatibility archive. The surrounding C block compressor continues to pass its existing entropy workspace and Huffman-table state through the unchanged internal ABI. The port preserves literal header choices, compression-gain decisions, repeat-table transitions, and 1X/4X encoder selection. It lets ordinary C compression paths exercise Rust code without widening this commit into the remaining frame compressor. Test Plan: - cargo clippy, cargo clippy --benches, cargo clippy --tests, and nightly fmt - cargo test --all-targets (94 passed) and feature-only cargo checks - native fuzzer, zstreamtest, invalidDictionaries, and CLI round trip - compare literals and next-table bytes with pristine C on x86_64 and i686 Refs: rust/README.md compression-primitives map
This commit is contained in:
@@ -8,228 +8,10 @@
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include "zstd_compress_literals.h"
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Debug Traces
|
||||
****************************************************************/
|
||||
#if DEBUGLEVEL >= 2
|
||||
|
||||
static size_t showHexa(const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* const ip = (const BYTE*)src;
|
||||
size_t u;
|
||||
for (u=0; u<srcSize; u++) {
|
||||
RAWLOG(5, " %02X", ip[u]); (void)ip;
|
||||
}
|
||||
RAWLOG(5, " \n");
|
||||
return srcSize;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Literals compression - special cases
|
||||
****************************************************************/
|
||||
size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
|
||||
|
||||
DEBUGLOG(5, "ZSTD_noCompressLiterals: srcSize=%zu, dstCapacity=%zu", srcSize, dstCapacity);
|
||||
|
||||
RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, "");
|
||||
|
||||
switch(flSize)
|
||||
{
|
||||
case 1: /* 2 - 1 - 5 */
|
||||
ostart[0] = (BYTE)((U32)set_basic + (srcSize<<3));
|
||||
break;
|
||||
case 2: /* 2 - 2 - 12 */
|
||||
MEM_writeLE16(ostart, (U16)((U32)set_basic + (1<<2) + (srcSize<<4)));
|
||||
break;
|
||||
case 3: /* 2 - 2 - 20 */
|
||||
MEM_writeLE32(ostart, (U32)((U32)set_basic + (3<<2) + (srcSize<<4)));
|
||||
break;
|
||||
default: /* not necessary : flSize is {1,2,3} */
|
||||
assert(0);
|
||||
}
|
||||
|
||||
ZSTD_memcpy(ostart + flSize, src, srcSize);
|
||||
DEBUGLOG(5, "Raw (uncompressed) literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize));
|
||||
return srcSize + flSize;
|
||||
}
|
||||
|
||||
static int allBytesIdentical(const void* src, size_t srcSize)
|
||||
{
|
||||
assert(srcSize >= 1);
|
||||
assert(src != NULL);
|
||||
{ const BYTE b = ((const BYTE*)src)[0];
|
||||
size_t p;
|
||||
for (p=1; p<srcSize; p++) {
|
||||
if (((const BYTE*)src)[p] != b) return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
U32 const flSize = 1 + (srcSize>31) + (srcSize>4095);
|
||||
|
||||
assert(dstCapacity >= 4); (void)dstCapacity;
|
||||
assert(allBytesIdentical(src, srcSize));
|
||||
|
||||
switch(flSize)
|
||||
{
|
||||
case 1: /* 2 - 1 - 5 */
|
||||
ostart[0] = (BYTE)((U32)set_rle + (srcSize<<3));
|
||||
break;
|
||||
case 2: /* 2 - 2 - 12 */
|
||||
MEM_writeLE16(ostart, (U16)((U32)set_rle + (1<<2) + (srcSize<<4)));
|
||||
break;
|
||||
case 3: /* 2 - 2 - 20 */
|
||||
MEM_writeLE32(ostart, (U32)((U32)set_rle + (3<<2) + (srcSize<<4)));
|
||||
break;
|
||||
default: /* not necessary : flSize is {1,2,3} */
|
||||
assert(0);
|
||||
}
|
||||
|
||||
ostart[flSize] = *(const BYTE*)src;
|
||||
DEBUGLOG(5, "RLE : Repeated Literal (%02X: %u times) -> %u bytes encoded", ((const BYTE*)src)[0], (U32)srcSize, (U32)flSize + 1);
|
||||
return flSize+1;
|
||||
}
|
||||
|
||||
/* ZSTD_minLiteralsToCompress() :
|
||||
* returns minimal amount of literals
|
||||
* for literal compression to even be attempted.
|
||||
* Minimum is made tighter as compression strategy increases.
|
||||
/*
|
||||
* Literals compression is implemented in
|
||||
* rust/src/zstd_compress_literals.rs. Keep this declaration-compatible
|
||||
* translation unit so every native build continues to use the existing
|
||||
* internal header and links the Rust ABI exports.
|
||||
*/
|
||||
static size_t
|
||||
ZSTD_minLiteralsToCompress(ZSTD_strategy strategy, HUF_repeat huf_repeat)
|
||||
{
|
||||
assert((int)strategy >= 0);
|
||||
assert((int)strategy <= 9);
|
||||
/* btultra2 : min 8 bytes;
|
||||
* then 2x larger for each successive compression strategy
|
||||
* max threshold 64 bytes */
|
||||
{ int const shift = MIN(9-(int)strategy, 3);
|
||||
size_t const mintc = (huf_repeat == HUF_repeat_valid) ? 6 : (size_t)8 << shift;
|
||||
DEBUGLOG(7, "minLiteralsToCompress = %zu", mintc);
|
||||
return mintc;
|
||||
}
|
||||
}
|
||||
|
||||
size_t ZSTD_compressLiterals (
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
void* entropyWorkspace, size_t entropyWorkspaceSize,
|
||||
const ZSTD_hufCTables_t* prevHuf,
|
||||
ZSTD_hufCTables_t* nextHuf,
|
||||
ZSTD_strategy strategy,
|
||||
int disableLiteralCompression,
|
||||
int suspectUncompressible,
|
||||
int bmi2)
|
||||
{
|
||||
size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
U32 singleStream = srcSize < 256;
|
||||
SymbolEncodingType_e hType = set_compressed;
|
||||
size_t cLitSize;
|
||||
|
||||
DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i, srcSize=%u, dstCapacity=%zu)",
|
||||
disableLiteralCompression, (U32)srcSize, dstCapacity);
|
||||
|
||||
DEBUGLOG(6, "Completed literals listing (%zu bytes)", showHexa(src, srcSize));
|
||||
|
||||
/* Prepare nextEntropy assuming reusing the existing table */
|
||||
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
||||
|
||||
if (disableLiteralCompression)
|
||||
return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
|
||||
|
||||
/* if too small, don't even attempt compression (speed opt) */
|
||||
if (srcSize < ZSTD_minLiteralsToCompress(strategy, prevHuf->repeatMode))
|
||||
return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
|
||||
|
||||
RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression");
|
||||
{ HUF_repeat repeat = prevHuf->repeatMode;
|
||||
int const flags = 0
|
||||
| (bmi2 ? HUF_flags_bmi2 : 0)
|
||||
| (strategy < ZSTD_lazy && srcSize <= 1024 ? HUF_flags_preferRepeat : 0)
|
||||
| (strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_flags_optimalDepth : 0)
|
||||
| (suspectUncompressible ? HUF_flags_suspectUncompressible : 0);
|
||||
|
||||
typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int);
|
||||
huf_compress_f huf_compress;
|
||||
if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
|
||||
huf_compress = singleStream ? HUF_compress1X_repeat : HUF_compress4X_repeat;
|
||||
cLitSize = huf_compress(ostart+lhSize, dstCapacity-lhSize,
|
||||
src, srcSize,
|
||||
HUF_SYMBOLVALUE_MAX, LitHufLog,
|
||||
entropyWorkspace, entropyWorkspaceSize,
|
||||
(HUF_CElt*)nextHuf->CTable,
|
||||
&repeat, flags);
|
||||
DEBUGLOG(5, "%zu literals compressed into %zu bytes (before header)", srcSize, cLitSize);
|
||||
if (repeat != HUF_repeat_none) {
|
||||
/* reused the existing table */
|
||||
DEBUGLOG(5, "reusing statistics from previous huffman block");
|
||||
hType = set_repeat;
|
||||
}
|
||||
}
|
||||
|
||||
{ size_t const minGain = ZSTD_minGain(srcSize, strategy);
|
||||
if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) {
|
||||
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
||||
return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
|
||||
} }
|
||||
if (cLitSize==1) {
|
||||
/* A return value of 1 signals that the alphabet consists of a single symbol.
|
||||
* However, in some rare circumstances, it could be the compressed size (a single byte).
|
||||
* For that outcome to have a chance to happen, it's necessary that `srcSize < 8`.
|
||||
* (it's also necessary to not generate statistics).
|
||||
* Therefore, in such a case, actively check that all bytes are identical. */
|
||||
if ((srcSize >= 8) || allBytesIdentical(src, srcSize)) {
|
||||
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
||||
return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
|
||||
} }
|
||||
|
||||
if (hType == set_compressed) {
|
||||
/* using a newly constructed table */
|
||||
nextHuf->repeatMode = HUF_repeat_check;
|
||||
}
|
||||
|
||||
/* Build header */
|
||||
switch(lhSize)
|
||||
{
|
||||
case 3: /* 2 - 2 - 10 - 10 */
|
||||
if (!singleStream) assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
|
||||
{ U32 const lhc = hType + ((U32)(!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14);
|
||||
MEM_writeLE24(ostart, lhc);
|
||||
break;
|
||||
}
|
||||
case 4: /* 2 - 2 - 14 - 14 */
|
||||
assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
|
||||
{ U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18);
|
||||
MEM_writeLE32(ostart, lhc);
|
||||
break;
|
||||
}
|
||||
case 5: /* 2 - 2 - 18 - 18 */
|
||||
assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
|
||||
{ U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22);
|
||||
MEM_writeLE32(ostart, lhc);
|
||||
ostart[4] = (BYTE)(cLitSize >> 10);
|
||||
break;
|
||||
}
|
||||
default: /* not possible : lhSize is {3,4,5} */
|
||||
assert(0);
|
||||
}
|
||||
DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)srcSize, (U32)(lhSize+cLitSize));
|
||||
return lhSize+cLitSize;
|
||||
}
|
||||
#include "zstd_compress_literals.h"
|
||||
|
||||
@@ -29,6 +29,8 @@ zstd ABI:
|
||||
- Compression primitives
|
||||
- `hist` counts byte frequencies for FSE and Huffman compression.
|
||||
- `zstd_presplit` chooses split points for full compression blocks.
|
||||
- `zstd_compress_literals` emits raw, RLE, and Huffman literal sections
|
||||
while preserving the compressor's Huffman-table repeat state.
|
||||
- Runtime support
|
||||
- `threading` provides platform pthread wrappers required by zstd headers.
|
||||
- `pool` implements the bounded worker pool used by multithreaded compression.
|
||||
|
||||
@@ -21,6 +21,8 @@ pub mod pool;
|
||||
pub mod threading;
|
||||
pub mod xxhash;
|
||||
pub mod zstd_common;
|
||||
#[cfg(feature = "compression")]
|
||||
pub mod zstd_compress_literals;
|
||||
#[cfg(feature = "decompression")]
|
||||
pub mod zstd_ddict;
|
||||
#[cfg(feature = "compression")]
|
||||
|
||||
@@ -0,0 +1,536 @@
|
||||
#![allow(non_snake_case)]
|
||||
|
||||
//! Compression of a block's literals section.
|
||||
//!
|
||||
//! This is a direct ABI translation of `zstd_compress_literals.c`. The
|
||||
//! surrounding block compressor remains in C during the migration, so the
|
||||
//! Huffman table wrapper deliberately uses the exact C layout and the caller
|
||||
//! continues to own the entropy workspace.
|
||||
|
||||
use crate::common::{LIT_HUF_LOG, MIN_LITERALS_FOR_4_STREAMS};
|
||||
use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
|
||||
use crate::huf_compress::{HUF_compress1X_repeat, HUF_compress4X_repeat};
|
||||
use crate::mem::{MEM_writeLE16, MEM_writeLE24, MEM_writeLE32};
|
||||
use std::ffi::c_void;
|
||||
use std::mem::size_of;
|
||||
use std::os::raw::c_int;
|
||||
use std::ptr;
|
||||
|
||||
const HUF_SYMBOLVALUE_MAX: u32 = 255;
|
||||
const HUF_CTABLE_SIZE_ST: usize = HUF_SYMBOLVALUE_MAX as usize + 2;
|
||||
|
||||
const HUF_REPEAT_NONE: c_int = 0;
|
||||
const HUF_REPEAT_CHECK: c_int = 1;
|
||||
const HUF_REPEAT_VALID: c_int = 2;
|
||||
|
||||
const HUF_FLAGS_BMI2: c_int = 1 << 0;
|
||||
const HUF_FLAGS_OPTIMAL_DEPTH: c_int = 1 << 1;
|
||||
const HUF_FLAGS_PREFER_REPEAT: c_int = 1 << 2;
|
||||
const HUF_FLAGS_SUSPECT_UNCOMPRESSIBLE: c_int = 1 << 3;
|
||||
|
||||
const ZSTD_LAZY: c_int = 4;
|
||||
const ZSTD_BTULTRA: c_int = 8;
|
||||
|
||||
const SET_BASIC: u32 = 0;
|
||||
const SET_RLE: u32 = 1;
|
||||
const SET_COMPRESSED: u32 = 2;
|
||||
const SET_REPEAT: u32 = 3;
|
||||
|
||||
/// ABI-compatible `ZSTD_hufCTables_t` from `zstd_compress_internal.h`.
|
||||
///
|
||||
/// `HUF_CElt` is `size_t` in C, and `HUF_repeat` is the platform C enum ABI
|
||||
/// (`int` for the supported C compilers). Keeping this type explicit is
|
||||
/// necessary because C compression code passes it across the language
|
||||
/// boundary for every compressed block.
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct ZSTD_hufCTables_t {
|
||||
pub CTable: [usize; HUF_CTABLE_SIZE_ST],
|
||||
pub repeatMode: c_int,
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn literal_header_size(src_size: usize) -> usize {
|
||||
1 + usize::from(src_size > 31) + usize::from(src_size > 4095)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn copy_bytes(dst: *mut u8, src: *const u8, size: usize) {
|
||||
// C's ZSTD_memcpy() permits the zero-length call used by the empty
|
||||
// literals path. Avoid forming a Rust copy from a potentially-null
|
||||
// zero-length source while preserving the byte-for-byte result.
|
||||
if size != 0 {
|
||||
unsafe { ptr::copy_nonoverlapping(src, dst, size) };
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn copy_huf_tables(dst: *mut ZSTD_hufCTables_t, src: *const ZSTD_hufCTables_t) {
|
||||
// Copy as bytes, not as a Rust struct: C's memcpy preserves the four tail
|
||||
// padding bytes that this layout has on 64-bit targets. Some C callers
|
||||
// copy or compare the complete entropy state, so leaving those bytes from
|
||||
// `nextHuf` would not be ABI-equivalent even though the named fields match.
|
||||
unsafe {
|
||||
ptr::copy(
|
||||
src.cast::<u8>(),
|
||||
dst.cast::<u8>(),
|
||||
size_of::<ZSTD_hufCTables_t>(),
|
||||
)
|
||||
};
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn all_bytes_identical(src: *const u8, src_size: usize) -> bool {
|
||||
if src_size == 0 || src.is_null() {
|
||||
return false;
|
||||
}
|
||||
let first = unsafe { *src };
|
||||
for index in 1..src_size {
|
||||
if unsafe { *src.add(index) } != first {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Writes an uncompressed literals section.
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_noCompressLiterals(
|
||||
dst: *mut c_void,
|
||||
dst_capacity: usize,
|
||||
src: *const c_void,
|
||||
src_size: usize,
|
||||
) -> usize {
|
||||
let fl_size = literal_header_size(src_size);
|
||||
let needed = match src_size.checked_add(fl_size) {
|
||||
Some(size) => size,
|
||||
None => return ERROR(ZstdErrorCode::DstSizeTooSmall),
|
||||
};
|
||||
if needed > dst_capacity {
|
||||
return ERROR(ZstdErrorCode::DstSizeTooSmall);
|
||||
}
|
||||
|
||||
let output = dst.cast::<u8>();
|
||||
match fl_size {
|
||||
1 => unsafe {
|
||||
*output = SET_BASIC.wrapping_add((src_size as u32).wrapping_shl(3)) as u8;
|
||||
},
|
||||
2 => unsafe {
|
||||
MEM_writeLE16(
|
||||
output.cast::<c_void>(),
|
||||
SET_BASIC
|
||||
.wrapping_add(1 << 2)
|
||||
.wrapping_add((src_size as u32).wrapping_shl(4)) as u16,
|
||||
);
|
||||
},
|
||||
3 => unsafe {
|
||||
MEM_writeLE32(
|
||||
output.cast::<c_void>(),
|
||||
SET_BASIC
|
||||
.wrapping_add(3 << 2)
|
||||
.wrapping_add((src_size as u32).wrapping_shl(4)),
|
||||
);
|
||||
},
|
||||
_ => unreachable!("literal header size is always one to three bytes"),
|
||||
}
|
||||
unsafe { copy_bytes(output.add(fl_size), src.cast::<u8>(), src_size) };
|
||||
needed
|
||||
}
|
||||
|
||||
/// Writes an RLE literals section.
|
||||
///
|
||||
/// Like the C entry point, callers must provide at least four output bytes and
|
||||
/// a non-empty input consisting entirely of one byte.
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_compressRleLiteralsBlock(
|
||||
dst: *mut c_void,
|
||||
dst_capacity: usize,
|
||||
src: *const c_void,
|
||||
src_size: usize,
|
||||
) -> usize {
|
||||
debug_assert!(dst_capacity >= 4);
|
||||
debug_assert!(all_bytes_identical(src.cast::<u8>(), src_size));
|
||||
|
||||
let fl_size = literal_header_size(src_size);
|
||||
let output = dst.cast::<u8>();
|
||||
match fl_size {
|
||||
1 => unsafe {
|
||||
*output = SET_RLE.wrapping_add((src_size as u32).wrapping_shl(3)) as u8;
|
||||
},
|
||||
2 => unsafe {
|
||||
MEM_writeLE16(
|
||||
output.cast::<c_void>(),
|
||||
SET_RLE
|
||||
.wrapping_add(1 << 2)
|
||||
.wrapping_add((src_size as u32).wrapping_shl(4)) as u16,
|
||||
);
|
||||
},
|
||||
3 => unsafe {
|
||||
MEM_writeLE32(
|
||||
output.cast::<c_void>(),
|
||||
SET_RLE
|
||||
.wrapping_add(3 << 2)
|
||||
.wrapping_add((src_size as u32).wrapping_shl(4)),
|
||||
);
|
||||
},
|
||||
_ => unreachable!("literal header size is always one to three bytes"),
|
||||
}
|
||||
unsafe { *output.add(fl_size) = *src.cast::<u8>() };
|
||||
fl_size + 1
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn min_literals_to_compress(strategy: c_int, huf_repeat: c_int) -> usize {
|
||||
// ZSTD_strategy is pre-validated by the C compression context. Clamp the
|
||||
// calculated shift nevertheless so an invalid direct FFI call cannot turn
|
||||
// into a Rust invalid shift; valid strategy values retain C's formula.
|
||||
let shift = (9 - strategy).clamp(0, 3) as u32;
|
||||
if huf_repeat == HUF_REPEAT_VALID {
|
||||
6
|
||||
} else {
|
||||
8usize << shift
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn min_gain(src_size: usize, strategy: c_int) -> usize {
|
||||
let min_log = if strategy >= ZSTD_BTULTRA {
|
||||
strategy.saturating_sub(1) as u32
|
||||
} else {
|
||||
6
|
||||
};
|
||||
// A valid ZSTD_strategy is at most 9. The clamp only gives invalid direct
|
||||
// FFI callers a defined result instead of allowing an invalid shift.
|
||||
(src_size >> min_log.min(usize::BITS - 1)) + 2
|
||||
}
|
||||
|
||||
/// Compresses a literals section with the previous Huffman table when useful.
|
||||
///
|
||||
/// The entropy workspace is passed straight into the HUF compressor. Its
|
||||
/// alignment and minimum size therefore retain the `HUF_compress*repeat()` C
|
||||
/// contract; this wrapper does not allocate a replacement workspace.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_compressLiterals(
|
||||
dst: *mut c_void,
|
||||
dst_capacity: usize,
|
||||
src: *const c_void,
|
||||
src_size: usize,
|
||||
entropy_workspace: *mut c_void,
|
||||
entropy_workspace_size: usize,
|
||||
prev_huf: *const ZSTD_hufCTables_t,
|
||||
next_huf: *mut ZSTD_hufCTables_t,
|
||||
strategy: c_int,
|
||||
disable_literal_compression: c_int,
|
||||
suspect_uncompressible: c_int,
|
||||
bmi2: c_int,
|
||||
) -> usize {
|
||||
let lh_size = 3 + usize::from(src_size >= 1024) + usize::from(src_size >= 16 * 1024);
|
||||
let output = dst.cast::<u8>();
|
||||
let mut single_stream = src_size < 256;
|
||||
let mut h_type = SET_COMPRESSED;
|
||||
|
||||
unsafe { copy_huf_tables(next_huf, prev_huf) };
|
||||
|
||||
if disable_literal_compression != 0 {
|
||||
return unsafe { ZSTD_noCompressLiterals(dst, dst_capacity, src, src_size) };
|
||||
}
|
||||
|
||||
let previous_repeat = unsafe { (*prev_huf).repeatMode };
|
||||
if src_size < min_literals_to_compress(strategy, previous_repeat) {
|
||||
return unsafe { ZSTD_noCompressLiterals(dst, dst_capacity, src, src_size) };
|
||||
}
|
||||
|
||||
if dst_capacity < lh_size + 1 {
|
||||
return ERROR(ZstdErrorCode::DstSizeTooSmall);
|
||||
}
|
||||
|
||||
let mut repeat = previous_repeat;
|
||||
let mut flags = 0;
|
||||
if bmi2 != 0 {
|
||||
flags |= HUF_FLAGS_BMI2;
|
||||
}
|
||||
if strategy < ZSTD_LAZY && src_size <= 1024 {
|
||||
flags |= HUF_FLAGS_PREFER_REPEAT;
|
||||
}
|
||||
if strategy >= ZSTD_BTULTRA {
|
||||
flags |= HUF_FLAGS_OPTIMAL_DEPTH;
|
||||
}
|
||||
if suspect_uncompressible != 0 {
|
||||
flags |= HUF_FLAGS_SUSPECT_UNCOMPRESSIBLE;
|
||||
}
|
||||
|
||||
if repeat == HUF_REPEAT_VALID && lh_size == 3 {
|
||||
single_stream = true;
|
||||
}
|
||||
|
||||
let c_lit_size = if single_stream {
|
||||
unsafe {
|
||||
HUF_compress1X_repeat(
|
||||
output.add(lh_size).cast::<c_void>(),
|
||||
dst_capacity - lh_size,
|
||||
src,
|
||||
src_size,
|
||||
HUF_SYMBOLVALUE_MAX,
|
||||
LIT_HUF_LOG as u32,
|
||||
entropy_workspace,
|
||||
entropy_workspace_size,
|
||||
(*next_huf).CTable.as_mut_ptr(),
|
||||
&mut repeat,
|
||||
flags,
|
||||
)
|
||||
}
|
||||
} else {
|
||||
unsafe {
|
||||
HUF_compress4X_repeat(
|
||||
output.add(lh_size).cast::<c_void>(),
|
||||
dst_capacity - lh_size,
|
||||
src,
|
||||
src_size,
|
||||
HUF_SYMBOLVALUE_MAX,
|
||||
LIT_HUF_LOG as u32,
|
||||
entropy_workspace,
|
||||
entropy_workspace_size,
|
||||
(*next_huf).CTable.as_mut_ptr(),
|
||||
&mut repeat,
|
||||
flags,
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
if c_lit_size == 0
|
||||
|| c_lit_size >= src_size.saturating_sub(min_gain(src_size, strategy))
|
||||
|| ERR_isError(c_lit_size)
|
||||
{
|
||||
unsafe { copy_huf_tables(next_huf, prev_huf) };
|
||||
return unsafe { ZSTD_noCompressLiterals(dst, dst_capacity, src, src_size) };
|
||||
}
|
||||
|
||||
if c_lit_size == 1 && (src_size >= 8 || all_bytes_identical(src.cast::<u8>(), src_size)) {
|
||||
unsafe { copy_huf_tables(next_huf, prev_huf) };
|
||||
return unsafe { ZSTD_compressRleLiteralsBlock(dst, dst_capacity, src, src_size) };
|
||||
}
|
||||
|
||||
if repeat != HUF_REPEAT_NONE {
|
||||
h_type = SET_REPEAT;
|
||||
}
|
||||
if h_type == SET_COMPRESSED {
|
||||
unsafe { (*next_huf).repeatMode = HUF_REPEAT_CHECK };
|
||||
}
|
||||
|
||||
match lh_size {
|
||||
3 => {
|
||||
debug_assert!(single_stream || src_size >= MIN_LITERALS_FOR_4_STREAMS);
|
||||
let header = h_type
|
||||
.wrapping_add(u32::from(!single_stream) << 2)
|
||||
.wrapping_add((src_size as u32).wrapping_shl(4))
|
||||
.wrapping_add((c_lit_size as u32).wrapping_shl(14));
|
||||
unsafe { MEM_writeLE24(output.cast::<c_void>(), header) };
|
||||
}
|
||||
4 => {
|
||||
debug_assert!(src_size >= MIN_LITERALS_FOR_4_STREAMS);
|
||||
let header = h_type
|
||||
.wrapping_add(2 << 2)
|
||||
.wrapping_add((src_size as u32).wrapping_shl(4))
|
||||
.wrapping_add((c_lit_size as u32).wrapping_shl(18));
|
||||
unsafe { MEM_writeLE32(output.cast::<c_void>(), header) };
|
||||
}
|
||||
5 => {
|
||||
debug_assert!(src_size >= MIN_LITERALS_FOR_4_STREAMS);
|
||||
let header = h_type
|
||||
.wrapping_add(3 << 2)
|
||||
.wrapping_add((src_size as u32).wrapping_shl(4))
|
||||
.wrapping_add((c_lit_size as u32).wrapping_shl(22));
|
||||
unsafe {
|
||||
MEM_writeLE32(output.cast::<c_void>(), header);
|
||||
*output.add(4) = (c_lit_size >> 10) as u8;
|
||||
}
|
||||
}
|
||||
_ => unreachable!("literal header size is always three to five bytes"),
|
||||
}
|
||||
|
||||
lh_size + c_lit_size
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::mem::{align_of, size_of};
|
||||
|
||||
#[test]
|
||||
fn huf_table_layout_matches_the_c_header() {
|
||||
assert_eq!(align_of::<ZSTD_hufCTables_t>(), align_of::<usize>());
|
||||
assert_eq!(
|
||||
size_of::<ZSTD_hufCTables_t>(),
|
||||
HUF_CTABLE_SIZE_ST * size_of::<usize>() + size_of::<usize>(),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_literals_headers_match_the_format() {
|
||||
let source = [0xABu8; 4096];
|
||||
let mut output = [0u8; 4099];
|
||||
|
||||
let size = unsafe {
|
||||
ZSTD_noCompressLiterals(
|
||||
output.as_mut_ptr().cast(),
|
||||
output.len(),
|
||||
source.as_ptr().cast(),
|
||||
31,
|
||||
)
|
||||
};
|
||||
assert_eq!(size, 32);
|
||||
assert_eq!(output[0], 31 << 3);
|
||||
assert_eq!(&output[1..size], &source[..31]);
|
||||
|
||||
let size = unsafe {
|
||||
ZSTD_noCompressLiterals(
|
||||
output.as_mut_ptr().cast(),
|
||||
output.len(),
|
||||
source.as_ptr().cast(),
|
||||
32,
|
||||
)
|
||||
};
|
||||
assert_eq!(size, 34);
|
||||
assert_eq!(&output[..2], &[0x04, 0x02]);
|
||||
assert_eq!(&output[2..size], &source[..32]);
|
||||
|
||||
let size = unsafe {
|
||||
ZSTD_noCompressLiterals(
|
||||
output.as_mut_ptr().cast(),
|
||||
output.len(),
|
||||
source.as_ptr().cast(),
|
||||
source.len(),
|
||||
)
|
||||
};
|
||||
assert_eq!(size, 4099);
|
||||
assert_eq!(&output[..3], &[0x0C, 0x00, 0x01]);
|
||||
assert_eq!(&output[3..size], source.as_slice());
|
||||
|
||||
output.fill(0xA5);
|
||||
let error = unsafe {
|
||||
ZSTD_noCompressLiterals(
|
||||
output.as_mut_ptr().cast(),
|
||||
source.len() + 2,
|
||||
source.as_ptr().cast(),
|
||||
source.len(),
|
||||
)
|
||||
};
|
||||
assert_eq!(error, ERROR(ZstdErrorCode::DstSizeTooSmall));
|
||||
assert!(output.iter().all(|&byte| byte == 0xA5));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rle_literals_headers_match_the_format() {
|
||||
let source = [0x5Au8; 4096];
|
||||
let mut output = [0u8; 4];
|
||||
|
||||
let size = unsafe {
|
||||
ZSTD_compressRleLiteralsBlock(
|
||||
output.as_mut_ptr().cast(),
|
||||
output.len(),
|
||||
source.as_ptr().cast(),
|
||||
31,
|
||||
)
|
||||
};
|
||||
assert_eq!(size, 2);
|
||||
assert_eq!(&output[..size], &[0xF9, 0x5A]);
|
||||
|
||||
let size = unsafe {
|
||||
ZSTD_compressRleLiteralsBlock(
|
||||
output.as_mut_ptr().cast(),
|
||||
output.len(),
|
||||
source.as_ptr().cast(),
|
||||
source.len(),
|
||||
)
|
||||
};
|
||||
assert_eq!(size, 4);
|
||||
assert_eq!(&output[..size], &[0x0D, 0x00, 0x01, 0x5A]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn direct_paths_copy_the_previous_huf_state() {
|
||||
let mut previous = ZSTD_hufCTables_t {
|
||||
CTable: [0; HUF_CTABLE_SIZE_ST],
|
||||
repeatMode: HUF_REPEAT_VALID,
|
||||
};
|
||||
previous.CTable[23] = 0x1234_5678;
|
||||
let mut next = ZSTD_hufCTables_t {
|
||||
CTable: [usize::MAX; HUF_CTABLE_SIZE_ST],
|
||||
repeatMode: HUF_REPEAT_NONE,
|
||||
};
|
||||
let source = [1u8, 2, 3];
|
||||
let mut output = [0u8; 8];
|
||||
|
||||
let size = unsafe {
|
||||
ZSTD_compressLiterals(
|
||||
output.as_mut_ptr().cast(),
|
||||
output.len(),
|
||||
source.as_ptr().cast(),
|
||||
source.len(),
|
||||
ptr::null_mut(),
|
||||
0,
|
||||
&previous,
|
||||
&mut next,
|
||||
ZSTD_LAZY,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
};
|
||||
assert_eq!(size, 4);
|
||||
assert_eq!(&output[..size], &[24, 1, 2, 3]);
|
||||
assert_eq!(next.CTable[23], previous.CTable[23]);
|
||||
assert_eq!(next.repeatMode, previous.repeatMode);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn huffman_literals_use_the_c_table_layout() {
|
||||
const HUF_WORKSPACE_SIZE: usize = (8 << 10) + 512;
|
||||
|
||||
let mut source = [0u8; 1024];
|
||||
for (index, byte) in source.iter_mut().enumerate() {
|
||||
*byte = (index % 4) as u8;
|
||||
}
|
||||
let previous = ZSTD_hufCTables_t {
|
||||
CTable: [0; HUF_CTABLE_SIZE_ST],
|
||||
repeatMode: HUF_REPEAT_NONE,
|
||||
};
|
||||
let mut next = previous;
|
||||
let mut output = [0u8; 2048];
|
||||
let mut workspace = [0u64; HUF_WORKSPACE_SIZE / size_of::<u64>()];
|
||||
|
||||
let size = unsafe {
|
||||
ZSTD_compressLiterals(
|
||||
output.as_mut_ptr().cast(),
|
||||
output.len(),
|
||||
source.as_ptr().cast(),
|
||||
source.len(),
|
||||
workspace.as_mut_ptr().cast(),
|
||||
size_of::<[u64; HUF_WORKSPACE_SIZE / size_of::<u64>()]>(),
|
||||
&previous,
|
||||
&mut next,
|
||||
3,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
};
|
||||
|
||||
assert!(!ERR_isError(size));
|
||||
assert!(size < source.len());
|
||||
assert_eq!(output[0] & 3, SET_COMPRESSED as u8);
|
||||
assert_eq!(next.repeatMode, HUF_REPEAT_CHECK);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn min_literal_thresholds_match_strategies() {
|
||||
assert_eq!(min_literals_to_compress(1, HUF_REPEAT_NONE), 64);
|
||||
assert_eq!(min_literals_to_compress(6, HUF_REPEAT_NONE), 64);
|
||||
assert_eq!(min_literals_to_compress(8, HUF_REPEAT_NONE), 16);
|
||||
assert_eq!(min_literals_to_compress(9, HUF_REPEAT_NONE), 8);
|
||||
assert_eq!(min_literals_to_compress(1, HUF_REPEAT_VALID), 6);
|
||||
assert_eq!(min_gain(128, 1), 4);
|
||||
assert_eq!(min_gain(128, ZSTD_BTULTRA), 3);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user