ddidderr
bdb14b4fbf
Move sequence-store byte accounting, chunk derivation, and repcode resolution into the Rust compression module. Move MT raw-sequence buffer conversions into Rust while preserving the existing C adapters and ABI layouts. Test Plan:\n- cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression\n- cargo clippy --manifest-path rust/Cargo.toml\n- cargo clippy --manifest-path rust/Cargo.toml --benches\n- cargo clippy --manifest-path rust/Cargo.toml --tests\n- make -B -C lib -j2 lib\n- make -B -C tests -j2 test-cli-tests
2144 lines
74 KiB
Rust
2144 lines
74 KiB
Rust
#![allow(non_camel_case_types)]
|
|
#![allow(non_snake_case)]
|
|
|
|
//! Sequence statistics and seqStore entropy compression.
|
|
//!
|
|
//! This module translates the block entropy-statistics layer of
|
|
//! `zstd_compress.c`: `ZSTD_seqToCodes()`, `ZSTD_buildSequencesStatistics()`,
|
|
//! `ZSTD_entropyCompressSeqStore*()`, `ZSTD_buildBlockEntropyStats*()`, and
|
|
//! `ZSTD_copyBlockSequences()`. The `ZSTD_CCtx` and `ZSTD_CCtx_params`
|
|
//! layouts stay private to C: the C shims extract the sequence store, the
|
|
//! entropy-table leaves, and the two parameter scalars these paths read
|
|
//! (the compression strategy and the literals-compression switch). Block
|
|
//! dispatch, block splitting, and the block-size estimation heuristics also
|
|
//! remain in C for a later slice.
|
|
|
|
use crate::bits::ZSTD_highbit32;
|
|
use crate::common::{
|
|
DEFAULT_MAX_OFF, LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, LL_FSE_LOG, LONGNBSEQ, MAX_LL, MAX_ML,
|
|
MAX_OFF, MAX_SEQ, MINMATCH, ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, ML_FSE_LOG, OFF_FSE_LOG,
|
|
OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, REP_START_VALUE, ZSTD_MAX_FSE_HEADERS_SIZE,
|
|
ZSTD_MAX_HUF_HEADER_SIZE, ZSTD_REP_NUM,
|
|
};
|
|
use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
|
|
use crate::hist::{HIST_countFast_wksp, HIST_count_wksp};
|
|
use crate::huf_compress::{
|
|
HUF_buildCTable_wksp, HUF_estimateCompressedSize, HUF_optimalTableLog, HUF_validateCTable,
|
|
HUF_writeCTable_wksp,
|
|
};
|
|
use crate::mem::{MEM_32bits, MEM_readST, MEM_writeLE16};
|
|
use crate::zstd_compress_literals::{
|
|
copy_huf_tables, min_gain, ZSTD_compressLiterals, ZSTD_hufCTables_t,
|
|
};
|
|
use crate::zstd_compress_sequences::{
|
|
SeqDef, ZSTD_buildCTable, ZSTD_encodeSequences, ZSTD_selectEncodingType,
|
|
};
|
|
use std::ffi::c_void;
|
|
use std::mem::{size_of, size_of_val};
|
|
use std::os::raw::{c_int, c_uint};
|
|
use std::ptr;
|
|
|
|
const SET_BASIC: c_int = 0;
|
|
const SET_RLE: c_int = 1;
|
|
const SET_COMPRESSED: c_int = 2;
|
|
const SET_REPEAT: c_int = 3;
|
|
|
|
const FSE_REPEAT_NONE: c_int = 0;
|
|
|
|
const HUF_REPEAT_NONE: c_int = 0;
|
|
const HUF_REPEAT_CHECK: c_int = 1;
|
|
const HUF_REPEAT_VALID: c_int = 2;
|
|
|
|
const HUF_FLAGS_OPTIMAL_DEPTH: c_int = 1 << 1;
|
|
const HUF_OPTIMAL_DEPTH_THRESHOLD: c_int = 8; /* ZSTD_btultra */
|
|
|
|
const ZSTD_DEFAULT_DISALLOWED: c_int = 0;
|
|
const ZSTD_DEFAULT_ALLOWED: c_int = 1;
|
|
|
|
const HUF_SYMBOLVALUE_MAX: c_uint = 255;
|
|
const LIT_HUF_LOG: c_uint = 11;
|
|
const COMPRESS_LITERALS_SIZE_MIN: usize = 63;
|
|
const SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO: usize = 20;
|
|
|
|
const STREAM_ACCUMULATOR_MIN_32: u32 = 25;
|
|
const STREAM_ACCUMULATOR_MIN_64: u32 = 57;
|
|
|
|
const LL_DELTA_CODE: u32 = 19;
|
|
const ML_DELTA_CODE: u32 = 36;
|
|
|
|
pub(crate) const ZSTD_LLT_LITERAL_LENGTH: c_int = 1;
|
|
pub(crate) const ZSTD_LLT_MATCH_LENGTH: c_int = 2;
|
|
|
|
/// `FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)` from the C headers.
|
|
pub const OFF_CTABLE_SIZE: usize = 1 + (1 << (OFF_FSE_LOG - 1)) + ((MAX_OFF + 1) * 2);
|
|
/// `FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)` from the C headers.
|
|
pub const ML_CTABLE_SIZE: usize = 1 + (1 << (ML_FSE_LOG - 1)) + ((MAX_ML + 1) * 2);
|
|
/// `FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)` from the C headers.
|
|
pub const LL_CTABLE_SIZE: usize = 1 + (1 << (LL_FSE_LOG - 1)) + ((MAX_LL + 1) * 2);
|
|
|
|
/// ABI-compatible `SeqStore_t` leaf layout from `zstd_compress_internal.h`.
|
|
///
|
|
/// The C context itself remains opaque; its shims pass the `seqStore` member
|
|
/// directly, so this small, stable hot-path structure is the only sequence
|
|
/// storage representation crossing into Rust.
|
|
#[repr(C)]
|
|
pub struct SeqStore_t {
|
|
pub sequencesStart: *mut SeqDef,
|
|
pub sequences: *mut SeqDef,
|
|
pub litStart: *mut u8,
|
|
pub lit: *mut u8,
|
|
pub llCode: *mut u8,
|
|
pub mlCode: *mut u8,
|
|
pub ofCode: *mut u8,
|
|
pub maxNbSeq: usize,
|
|
pub maxNbLit: usize,
|
|
pub longLengthType: c_int,
|
|
pub longLengthPos: u32,
|
|
}
|
|
|
|
/// C's `ZSTD_fseCTables_t`. The table element type is `FSE_CTable`, an
|
|
/// `unsigned`, and the table lengths are the header macros expanded above.
|
|
#[repr(C)]
|
|
pub struct ZSTD_fseCTables_t {
|
|
pub offcodeCTable: [u32; OFF_CTABLE_SIZE],
|
|
pub matchlengthCTable: [u32; ML_CTABLE_SIZE],
|
|
pub litlengthCTable: [u32; LL_CTABLE_SIZE],
|
|
pub offcode_repeatMode: c_int,
|
|
pub matchlength_repeatMode: c_int,
|
|
pub litlength_repeatMode: c_int,
|
|
}
|
|
|
|
/// C's `ZSTD_entropyCTables_t`.
|
|
#[repr(C)]
|
|
pub struct ZSTD_entropyCTables_t {
|
|
pub huf: ZSTD_hufCTables_t,
|
|
pub fse: ZSTD_fseCTables_t,
|
|
}
|
|
|
|
/// C's `ZSTD_compressedBlockState_t`.
|
|
#[repr(C)]
|
|
pub struct ZSTD_compressedBlockState_t {
|
|
pub entropy: ZSTD_entropyCTables_t,
|
|
pub rep: [u32; ZSTD_REP_NUM],
|
|
}
|
|
|
|
/// C's `ZSTD_hufCTablesMetadata_t`.
|
|
#[repr(C)]
|
|
pub struct ZSTD_hufCTablesMetadata_t {
|
|
pub hType: c_int,
|
|
pub hufDesBuffer: [u8; ZSTD_MAX_HUF_HEADER_SIZE],
|
|
pub hufDesSize: usize,
|
|
}
|
|
|
|
/// C's `ZSTD_fseCTablesMetadata_t`.
|
|
#[repr(C)]
|
|
pub struct ZSTD_fseCTablesMetadata_t {
|
|
pub llType: c_int,
|
|
pub ofType: c_int,
|
|
pub mlType: c_int,
|
|
pub fseTablesBuffer: [u8; ZSTD_MAX_FSE_HEADERS_SIZE],
|
|
pub fseTablesSize: usize,
|
|
pub lastCountSize: usize,
|
|
}
|
|
|
|
/// C's `ZSTD_entropyCTablesMetadata_t`.
|
|
#[repr(C)]
|
|
pub struct ZSTD_entropyCTablesMetadata_t {
|
|
pub hufMetadata: ZSTD_hufCTablesMetadata_t,
|
|
pub fseMetadata: ZSTD_fseCTablesMetadata_t,
|
|
}
|
|
|
|
/// Public `ZSTD_Sequence` from `zstd.h`.
|
|
#[repr(C)]
|
|
#[derive(Clone, Copy)]
|
|
pub struct ZSTD_Sequence {
|
|
pub offset: u32,
|
|
pub litLength: u32,
|
|
pub matchLength: u32,
|
|
pub rep: u32,
|
|
}
|
|
|
|
/// Resets the state that is carried from one compressed block to the next.
|
|
///
|
|
/// This is the direct Rust leaf for C's
|
|
/// `ZSTD_reset_compressedBlockState()`. The caller must provide a valid
|
|
/// mutable compressed block state.
|
|
#[no_mangle]
|
|
pub unsafe extern "C" fn ZSTD_rust_resetCompressedBlockState(
|
|
block_state: *mut ZSTD_compressedBlockState_t,
|
|
) {
|
|
let block_state = unsafe { &mut *block_state };
|
|
block_state.rep.copy_from_slice(&REP_START_VALUE);
|
|
block_state.entropy.huf.repeatMode = HUF_REPEAT_NONE;
|
|
block_state.entropy.fse.offcode_repeatMode = FSE_REPEAT_NONE;
|
|
block_state.entropy.fse.matchlength_repeatMode = FSE_REPEAT_NONE;
|
|
block_state.entropy.fse.litlength_repeatMode = FSE_REPEAT_NONE;
|
|
}
|
|
|
|
/// Appends the final literals of a block to its sequence store.
|
|
///
|
|
/// The source and destination ranges must be valid and non-overlapping, as
|
|
/// required by the original C `memcpy` call. `last_ll_size == 0` preserves
|
|
/// the C pointer-update behavior without changing any bytes.
|
|
#[no_mangle]
|
|
pub unsafe extern "C" fn ZSTD_rust_storeLastLiterals(
|
|
seq_store: *mut SeqStore_t,
|
|
anchor: *const u8,
|
|
last_ll_size: usize,
|
|
) {
|
|
let seq_store = unsafe { &mut *seq_store };
|
|
unsafe {
|
|
ptr::copy_nonoverlapping(anchor, seq_store.lit, last_ll_size);
|
|
seq_store.lit = seq_store.lit.add(last_ll_size);
|
|
}
|
|
}
|
|
|
|
/// Resets the sequence and literal cursors for a new block.
|
|
#[no_mangle]
|
|
pub unsafe extern "C" fn ZSTD_rust_resetSeqStore(seq_store: *mut SeqStore_t) {
|
|
let seq_store = unsafe { &mut *seq_store };
|
|
seq_store.lit = seq_store.litStart;
|
|
seq_store.sequences = seq_store.sequencesStart;
|
|
seq_store.longLengthType = 0;
|
|
}
|
|
|
|
/// Returns the sum of all literal and match lengths in an external sequence
|
|
/// buffer, without looking for a block delimiter.
|
|
#[no_mangle]
|
|
pub unsafe extern "C" fn ZSTD_rust_fastSequenceLengthSum(
|
|
seq_buf: *const ZSTD_Sequence,
|
|
seq_buf_size: usize,
|
|
) -> usize {
|
|
let mut match_len_sum = 0usize;
|
|
let mut lit_len_sum = 0usize;
|
|
for index in 0..seq_buf_size {
|
|
let sequence = unsafe { *seq_buf.add(index) };
|
|
lit_len_sum = lit_len_sum.wrapping_add(sequence.litLength as usize);
|
|
match_len_sum = match_len_sum.wrapping_add(sequence.matchLength as usize);
|
|
}
|
|
lit_len_sum.wrapping_add(match_len_sum)
|
|
}
|
|
|
|
/// Returns whether all bytes in the input have the same value.
|
|
///
|
|
/// This keeps the original C helper's precondition that `src` points to at
|
|
/// least one readable byte, even when `length == 0`: the first byte is read
|
|
/// before the length checks.
|
|
#[no_mangle]
|
|
pub unsafe extern "C" fn ZSTD_rust_isRLE(src: *const u8, length: usize) -> c_int {
|
|
let value = unsafe { *src };
|
|
let value_st = (u64::from(value) * 0x0101_0101_0101_0101u64) as usize;
|
|
let unroll_size = size_of::<usize>() * 4;
|
|
let prefix_length = length & (unroll_size - 1);
|
|
|
|
if length == 1 {
|
|
return 1;
|
|
}
|
|
|
|
let mut index = 1;
|
|
while index < prefix_length {
|
|
if unsafe { *src.add(index) } != value {
|
|
return 0;
|
|
}
|
|
index += 1;
|
|
}
|
|
|
|
let mut index = prefix_length;
|
|
while index != length {
|
|
let mut unroll_offset = 0;
|
|
while unroll_offset < unroll_size {
|
|
let word = unsafe { MEM_readST(src.add(index + unroll_offset).cast()) };
|
|
if word != value_st {
|
|
return 0;
|
|
}
|
|
unroll_offset += size_of::<usize>();
|
|
}
|
|
index += unroll_size;
|
|
}
|
|
1
|
|
}
|
|
|
|
/// Heuristic used to decide whether a sequence store may describe an RLE
|
|
/// block. It intentionally only inspects the two pointer distances.
|
|
#[no_mangle]
|
|
pub unsafe extern "C" fn ZSTD_rust_maybeRLE(seq_store: *const SeqStore_t) -> c_int {
|
|
let seq_store = unsafe { &*seq_store };
|
|
let nb_seqs = unsafe { seq_store.sequences.offset_from(seq_store.sequencesStart) } as usize;
|
|
let nb_lits = unsafe { seq_store.lit.offset_from(seq_store.litStart) } as usize;
|
|
if nb_seqs < 4 && nb_lits < 10 {
|
|
1
|
|
} else {
|
|
0
|
|
}
|
|
}
|
|
|
|
/// Returns the literal bytes represented by a sequence store.
|
|
///
|
|
/// The stored `u16` literal length may represent one long literal through the
|
|
/// sequence store's side-band marker. This is the Rust leaf for C's
|
|
/// `ZSTD_countSeqStoreLiteralsBytes()` helper.
|
|
#[inline]
|
|
unsafe fn count_seq_store_literals_bytes(seq_store: *const SeqStore_t) -> usize {
|
|
let seq_store = unsafe { &*seq_store };
|
|
let nb_seqs = unsafe { seq_store.sequences.offset_from(seq_store.sequencesStart) } as usize;
|
|
let mut literals_bytes = 0usize;
|
|
for index in 0..nb_seqs {
|
|
let sequence = unsafe { *seq_store.sequencesStart.add(index) };
|
|
literals_bytes = literals_bytes.wrapping_add(sequence.litLength as usize);
|
|
if index == seq_store.longLengthPos as usize
|
|
&& seq_store.longLengthType == ZSTD_LLT_LITERAL_LENGTH
|
|
{
|
|
literals_bytes = literals_bytes.wrapping_add(0x10000);
|
|
}
|
|
}
|
|
literals_bytes
|
|
}
|
|
|
|
/// Returns the match bytes represented by a sequence store.
|
|
///
|
|
/// `mlBase` stores `matchLength - MINMATCH`; the side-band long-length marker
|
|
/// adds the same `0x10000` extension used by the original C helper.
|
|
#[no_mangle]
|
|
pub unsafe extern "C" fn ZSTD_rust_countSeqStoreLiteralsBytes(
|
|
seq_store: *const SeqStore_t,
|
|
) -> usize {
|
|
unsafe { count_seq_store_literals_bytes(seq_store) }
|
|
}
|
|
|
|
#[inline]
|
|
unsafe fn count_seq_store_match_bytes(seq_store: *const SeqStore_t) -> usize {
|
|
let seq_store = unsafe { &*seq_store };
|
|
let nb_seqs = unsafe { seq_store.sequences.offset_from(seq_store.sequencesStart) } as usize;
|
|
let mut match_bytes = 0usize;
|
|
for index in 0..nb_seqs {
|
|
let sequence = unsafe { *seq_store.sequencesStart.add(index) };
|
|
match_bytes = match_bytes.wrapping_add(sequence.mlBase as usize + MINMATCH);
|
|
if index == seq_store.longLengthPos as usize
|
|
&& seq_store.longLengthType == ZSTD_LLT_MATCH_LENGTH
|
|
{
|
|
match_bytes = match_bytes.wrapping_add(0x10000);
|
|
}
|
|
}
|
|
match_bytes
|
|
}
|
|
|
|
/// Returns the match bytes represented by a sequence store.
|
|
#[no_mangle]
|
|
pub unsafe extern "C" fn ZSTD_rust_countSeqStoreMatchBytes(seq_store: *const SeqStore_t) -> usize {
|
|
unsafe { count_seq_store_match_bytes(seq_store) }
|
|
}
|
|
|
|
/// Derives the sequence-store view for the half-open sequence range
|
|
/// `[start_idx, end_idx)`.
|
|
///
|
|
/// This is the Rust leaf for C's `ZSTD_deriveSeqStoreChunk()`. The original
|
|
/// store remains untouched; the result is a shallow copy whose pointers refer
|
|
/// into the original sequence, literal, and code buffers.
|
|
#[no_mangle]
|
|
pub unsafe extern "C" fn ZSTD_rust_deriveSeqStoreChunk(
|
|
result_seq_store: *mut SeqStore_t,
|
|
original_seq_store: *const SeqStore_t,
|
|
start_idx: usize,
|
|
end_idx: usize,
|
|
) {
|
|
let original_seq_store = unsafe { &*original_seq_store };
|
|
let result_seq_store = unsafe { &mut *result_seq_store };
|
|
*result_seq_store = unsafe { ptr::read(original_seq_store) };
|
|
|
|
if start_idx > 0 {
|
|
result_seq_store.sequences = unsafe { original_seq_store.sequencesStart.add(start_idx) };
|
|
let literals_bytes =
|
|
unsafe { count_seq_store_literals_bytes(std::ptr::addr_of!(*result_seq_store)) };
|
|
result_seq_store.litStart = unsafe { result_seq_store.litStart.add(literals_bytes) };
|
|
}
|
|
|
|
/* Move longLengthPos into the correct position if necessary. */
|
|
if original_seq_store.longLengthType != 0 {
|
|
if original_seq_store.longLengthPos as usize > end_idx
|
|
|| (original_seq_store.longLengthPos as usize) < start_idx
|
|
{
|
|
result_seq_store.longLengthType = 0;
|
|
} else {
|
|
result_seq_store.longLengthPos = original_seq_store
|
|
.longLengthPos
|
|
.wrapping_sub(start_idx as u32);
|
|
}
|
|
}
|
|
result_seq_store.sequencesStart = unsafe { original_seq_store.sequencesStart.add(start_idx) };
|
|
result_seq_store.sequences = unsafe { original_seq_store.sequencesStart.add(end_idx) };
|
|
|
|
let original_nb_sequences = unsafe {
|
|
original_seq_store
|
|
.sequences
|
|
.offset_from(original_seq_store.sequencesStart)
|
|
} as usize;
|
|
if end_idx == original_nb_sequences {
|
|
/* This accounts for possible last literals at the end of the block. */
|
|
debug_assert_eq!(result_seq_store.lit, original_seq_store.lit);
|
|
} else {
|
|
let literals_bytes =
|
|
unsafe { count_seq_store_literals_bytes(std::ptr::addr_of!(*result_seq_store)) };
|
|
result_seq_store.lit = unsafe { result_seq_store.litStart.add(literals_bytes) };
|
|
}
|
|
result_seq_store.llCode = unsafe { result_seq_store.llCode.add(start_idx) };
|
|
result_seq_store.mlCode = unsafe { result_seq_store.mlCode.add(start_idx) };
|
|
result_seq_store.ofCode = unsafe { result_seq_store.ofCode.add(start_idx) };
|
|
}
|
|
|
|
/// Resolves a stored repcode against the current raw-offset history.
|
|
///
|
|
/// This is the Rust leaf for C's `ZSTD_resolveRepcodeToRawOffset()`. The
|
|
/// caller must pass an `off_base` in the repcode range `1..=ZSTD_REP_NUM` and
|
|
/// an `ll0` value of zero or one.
|
|
#[no_mangle]
|
|
pub unsafe extern "C" fn ZSTD_rust_resolveRepcodeToRawOffset(
|
|
rep: *const u32,
|
|
off_base: u32,
|
|
ll0: u32,
|
|
) -> u32 {
|
|
debug_assert!((1..=ZSTD_REP_NUM as u32).contains(&off_base));
|
|
let adjusted_rep_code = off_base.wrapping_sub(1).wrapping_add(ll0);
|
|
let rep = unsafe { std::slice::from_raw_parts(rep, ZSTD_REP_NUM) };
|
|
if adjusted_rep_code == ZSTD_REP_NUM as u32 {
|
|
debug_assert_ne!(ll0, 0);
|
|
rep[0].wrapping_sub(1)
|
|
} else {
|
|
rep[adjusted_rep_code as usize]
|
|
}
|
|
}
|
|
|
|
/// C's `SeqCollector` leaf from `zstd_compress_internal.h`.
|
|
#[repr(C)]
|
|
pub struct SeqCollector {
|
|
pub collectSequences: c_int,
|
|
pub seqStart: *mut ZSTD_Sequence,
|
|
pub seqIndex: usize,
|
|
pub maxSequences: usize,
|
|
}
|
|
|
|
/// Result of `build_sequences_statistics()`; mirrors C's
|
|
/// `ZSTD_symbolEncodingTypeStats_t`, which no longer crosses the boundary.
|
|
pub(crate) struct SymbolEncodingTypeStats {
|
|
pub ll_type: u32,
|
|
pub off_type: u32,
|
|
pub ml_type: u32,
|
|
pub size: usize,
|
|
/// Accounts for a bug in zstd 1.3.4; see
|
|
/// `entropy_compress_seq_store_internal()`.
|
|
pub last_count_size: usize,
|
|
pub long_offsets: c_int,
|
|
}
|
|
|
|
/// C's `ZSTD_updateRep()` repeat-offset history update.
|
|
#[inline]
|
|
pub(crate) fn update_rep(reps: &mut [u32; 3], off_base: u32, literal_length_is_zero: bool) {
|
|
if off_base > 3 {
|
|
reps[2] = reps[1];
|
|
reps[1] = reps[0];
|
|
reps[0] = off_base - 3;
|
|
return;
|
|
}
|
|
let rep_code = off_base - 1 + u32::from(literal_length_is_zero);
|
|
if rep_code == 0 {
|
|
return;
|
|
}
|
|
let current_offset = if rep_code == 3 {
|
|
reps[0].wrapping_sub(1)
|
|
} else {
|
|
reps[rep_code as usize]
|
|
};
|
|
reps[2] = if rep_code >= 2 { reps[1] } else { reps[2] };
|
|
reps[1] = reps[0];
|
|
reps[0] = current_offset;
|
|
}
|
|
|
|
#[inline]
|
|
fn ll_code(lit_length: u32) -> u32 {
|
|
const LL_CODE: [u8; 64] = [
|
|
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20,
|
|
20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 23, 23, 23,
|
|
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
|
|
];
|
|
if lit_length > 63 {
|
|
ZSTD_highbit32(lit_length) + LL_DELTA_CODE
|
|
} else {
|
|
LL_CODE[lit_length as usize] as u32
|
|
}
|
|
}
|
|
|
|
#[inline]
|
|
fn ml_code(ml_base: u32) -> u32 {
|
|
const ML_CODE: [u8; 128] = [
|
|
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
|
|
25, 26, 27, 28, 29, 30, 31, 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
|
|
38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40,
|
|
40, 40, 40, 40, 40, 40, 40, 40, 40, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
|
|
41, 41, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
|
|
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
|
|
];
|
|
if ml_base > 127 {
|
|
ZSTD_highbit32(ml_base) + ML_DELTA_CODE
|
|
} else {
|
|
ML_CODE[ml_base as usize] as u32
|
|
}
|
|
}
|
|
|
|
#[inline]
|
|
unsafe fn copy_fse_tables(dst: *mut ZSTD_fseCTables_t, src: *const ZSTD_fseCTables_t) {
|
|
unsafe {
|
|
ptr::copy_nonoverlapping(
|
|
src.cast::<u8>(),
|
|
dst.cast::<u8>(),
|
|
size_of::<ZSTD_fseCTables_t>(),
|
|
)
|
|
};
|
|
}
|
|
|
|
/// Converts the stored lengths and distances into symbol codes.
|
|
///
|
|
/// Retains its original exported name: the C dictionary builder and the
|
|
/// decodecorpus test tool link against it directly.
|
|
#[no_mangle]
|
|
pub unsafe extern "C" fn ZSTD_seqToCodes(seq_store: *const SeqStore_t) -> c_int {
|
|
let store = unsafe { &*seq_store };
|
|
let sequences = store.sequencesStart;
|
|
let nb_seq = unsafe { store.sequences.offset_from(sequences) } as usize;
|
|
let mut long_offsets = 0;
|
|
debug_assert!(nb_seq <= store.maxNbSeq);
|
|
|
|
for index in 0..nb_seq {
|
|
let sequence = unsafe { sequences.add(index).read() };
|
|
let of_code = ZSTD_highbit32(sequence.offBase);
|
|
unsafe {
|
|
*store.llCode.add(index) = ll_code(sequence.litLength as u32) as u8;
|
|
*store.ofCode.add(index) = of_code as u8;
|
|
*store.mlCode.add(index) = ml_code(sequence.mlBase as u32) as u8;
|
|
}
|
|
debug_assert!(MEM_32bits() || of_code < STREAM_ACCUMULATOR_MIN_64);
|
|
if MEM_32bits() && of_code >= STREAM_ACCUMULATOR_MIN_32 {
|
|
long_offsets = 1;
|
|
}
|
|
}
|
|
if store.longLengthType == ZSTD_LLT_LITERAL_LENGTH {
|
|
unsafe { *store.llCode.add(store.longLengthPos as usize) = MAX_LL as u8 };
|
|
}
|
|
if store.longLengthType == ZSTD_LLT_MATCH_LENGTH {
|
|
unsafe { *store.mlCode.add(store.longLengthPos as usize) = MAX_ML as u8 };
|
|
}
|
|
long_offsets
|
|
}
|
|
|
|
/// C's `ZSTD_buildSequencesStatistics()`.
|
|
///
|
|
/// Returns the statistics with a zstd error code in the `size` field on
|
|
/// failure, and updates `next_entropy` as a side effect. `nb_seq` must be
|
|
/// greater than zero, and the entropy workspace must hold at least
|
|
/// `ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1) * sizeof(U32)` bytes.
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub(crate) unsafe fn build_sequences_statistics(
|
|
seq_store: *const SeqStore_t,
|
|
nb_seq: usize,
|
|
prev_entropy: *const ZSTD_fseCTables_t,
|
|
next_entropy: *mut ZSTD_fseCTables_t,
|
|
dst: *mut u8,
|
|
dst_capacity: usize,
|
|
strategy: c_int,
|
|
count_workspace: *mut c_uint,
|
|
entropy_workspace: *mut c_void,
|
|
entropy_wksp_size: usize,
|
|
) -> SymbolEncodingTypeStats {
|
|
let store = unsafe { &*seq_store };
|
|
let mut stats = SymbolEncodingTypeStats {
|
|
ll_type: 0,
|
|
off_type: 0,
|
|
ml_type: 0,
|
|
size: 0,
|
|
last_count_size: 0,
|
|
long_offsets: 0,
|
|
};
|
|
let mut written = 0usize;
|
|
|
|
/* convert length/distances into codes */
|
|
stats.long_offsets = unsafe { ZSTD_seqToCodes(seq_store) };
|
|
debug_assert!(nb_seq != 0); /* ZSTD_selectEncodingType() divides by nb_seq */
|
|
|
|
/* build CTable for Literal Lengths */
|
|
{
|
|
let mut max = MAX_LL as c_uint;
|
|
let most_frequent = unsafe {
|
|
HIST_countFast_wksp(
|
|
count_workspace,
|
|
&mut max,
|
|
store.llCode.cast::<c_void>(),
|
|
nb_seq,
|
|
entropy_workspace,
|
|
entropy_wksp_size,
|
|
)
|
|
}; /* can't fail */
|
|
unsafe { (*next_entropy).litlength_repeatMode = (*prev_entropy).litlength_repeatMode };
|
|
stats.ll_type = unsafe {
|
|
ZSTD_selectEncodingType(
|
|
ptr::addr_of_mut!((*next_entropy).litlength_repeatMode),
|
|
count_workspace,
|
|
max,
|
|
most_frequent,
|
|
nb_seq,
|
|
LL_FSE_LOG as c_uint,
|
|
ptr::addr_of!((*prev_entropy).litlengthCTable).cast::<u32>(),
|
|
LL_DEFAULT_NORM.as_ptr(),
|
|
LL_DEFAULT_NORM_LOG,
|
|
ZSTD_DEFAULT_ALLOWED,
|
|
strategy,
|
|
)
|
|
} as u32;
|
|
let count_size = unsafe {
|
|
ZSTD_buildCTable(
|
|
dst.add(written).cast::<c_void>(),
|
|
dst_capacity - written,
|
|
ptr::addr_of_mut!((*next_entropy).litlengthCTable).cast::<u32>(),
|
|
LL_FSE_LOG as u32,
|
|
stats.ll_type as c_int,
|
|
count_workspace,
|
|
max,
|
|
store.llCode,
|
|
nb_seq,
|
|
LL_DEFAULT_NORM.as_ptr(),
|
|
LL_DEFAULT_NORM_LOG,
|
|
MAX_LL as u32,
|
|
ptr::addr_of!((*prev_entropy).litlengthCTable).cast::<u32>(),
|
|
size_of::<[u32; LL_CTABLE_SIZE]>(),
|
|
entropy_workspace,
|
|
entropy_wksp_size,
|
|
)
|
|
};
|
|
if ERR_isError(count_size) {
|
|
stats.size = count_size;
|
|
return stats;
|
|
}
|
|
if stats.ll_type == SET_COMPRESSED as u32 {
|
|
stats.last_count_size = count_size;
|
|
}
|
|
written += count_size;
|
|
}
|
|
/* build CTable for Offsets */
|
|
{
|
|
let mut max = MAX_OFF as c_uint;
|
|
let most_frequent = unsafe {
|
|
HIST_countFast_wksp(
|
|
count_workspace,
|
|
&mut max,
|
|
store.ofCode.cast::<c_void>(),
|
|
nb_seq,
|
|
entropy_workspace,
|
|
entropy_wksp_size,
|
|
)
|
|
}; /* can't fail */
|
|
/* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */
|
|
let default_policy = if max <= DEFAULT_MAX_OFF as c_uint {
|
|
ZSTD_DEFAULT_ALLOWED
|
|
} else {
|
|
ZSTD_DEFAULT_DISALLOWED
|
|
};
|
|
unsafe { (*next_entropy).offcode_repeatMode = (*prev_entropy).offcode_repeatMode };
|
|
stats.off_type = unsafe {
|
|
ZSTD_selectEncodingType(
|
|
ptr::addr_of_mut!((*next_entropy).offcode_repeatMode),
|
|
count_workspace,
|
|
max,
|
|
most_frequent,
|
|
nb_seq,
|
|
OFF_FSE_LOG as c_uint,
|
|
ptr::addr_of!((*prev_entropy).offcodeCTable).cast::<u32>(),
|
|
OF_DEFAULT_NORM.as_ptr(),
|
|
OF_DEFAULT_NORM_LOG,
|
|
default_policy,
|
|
strategy,
|
|
)
|
|
} as u32;
|
|
let count_size = unsafe {
|
|
ZSTD_buildCTable(
|
|
dst.add(written).cast::<c_void>(),
|
|
dst_capacity - written,
|
|
ptr::addr_of_mut!((*next_entropy).offcodeCTable).cast::<u32>(),
|
|
OFF_FSE_LOG as u32,
|
|
stats.off_type as c_int,
|
|
count_workspace,
|
|
max,
|
|
store.ofCode,
|
|
nb_seq,
|
|
OF_DEFAULT_NORM.as_ptr(),
|
|
OF_DEFAULT_NORM_LOG,
|
|
DEFAULT_MAX_OFF as u32,
|
|
ptr::addr_of!((*prev_entropy).offcodeCTable).cast::<u32>(),
|
|
size_of::<[u32; OFF_CTABLE_SIZE]>(),
|
|
entropy_workspace,
|
|
entropy_wksp_size,
|
|
)
|
|
};
|
|
if ERR_isError(count_size) {
|
|
stats.size = count_size;
|
|
return stats;
|
|
}
|
|
if stats.off_type == SET_COMPRESSED as u32 {
|
|
stats.last_count_size = count_size;
|
|
}
|
|
written += count_size;
|
|
}
|
|
/* build CTable for MatchLengths */
|
|
{
|
|
let mut max = MAX_ML as c_uint;
|
|
let most_frequent = unsafe {
|
|
HIST_countFast_wksp(
|
|
count_workspace,
|
|
&mut max,
|
|
store.mlCode.cast::<c_void>(),
|
|
nb_seq,
|
|
entropy_workspace,
|
|
entropy_wksp_size,
|
|
)
|
|
}; /* can't fail */
|
|
unsafe { (*next_entropy).matchlength_repeatMode = (*prev_entropy).matchlength_repeatMode };
|
|
stats.ml_type = unsafe {
|
|
ZSTD_selectEncodingType(
|
|
ptr::addr_of_mut!((*next_entropy).matchlength_repeatMode),
|
|
count_workspace,
|
|
max,
|
|
most_frequent,
|
|
nb_seq,
|
|
ML_FSE_LOG as c_uint,
|
|
ptr::addr_of!((*prev_entropy).matchlengthCTable).cast::<u32>(),
|
|
ML_DEFAULT_NORM.as_ptr(),
|
|
ML_DEFAULT_NORM_LOG,
|
|
ZSTD_DEFAULT_ALLOWED,
|
|
strategy,
|
|
)
|
|
} as u32;
|
|
let count_size = unsafe {
|
|
ZSTD_buildCTable(
|
|
dst.add(written).cast::<c_void>(),
|
|
dst_capacity - written,
|
|
ptr::addr_of_mut!((*next_entropy).matchlengthCTable).cast::<u32>(),
|
|
ML_FSE_LOG as u32,
|
|
stats.ml_type as c_int,
|
|
count_workspace,
|
|
max,
|
|
store.mlCode,
|
|
nb_seq,
|
|
ML_DEFAULT_NORM.as_ptr(),
|
|
ML_DEFAULT_NORM_LOG,
|
|
MAX_ML as u32,
|
|
ptr::addr_of!((*prev_entropy).matchlengthCTable).cast::<u32>(),
|
|
size_of::<[u32; ML_CTABLE_SIZE]>(),
|
|
entropy_workspace,
|
|
entropy_wksp_size,
|
|
)
|
|
};
|
|
if ERR_isError(count_size) {
|
|
stats.size = count_size;
|
|
return stats;
|
|
}
|
|
if stats.ml_type == SET_COMPRESSED as u32 {
|
|
stats.last_count_size = count_size;
|
|
}
|
|
written += count_size;
|
|
}
|
|
stats.size = written;
|
|
stats
|
|
}
|
|
|
|
/// C's `ZSTD_entropyCompressSeqStore_internal()`: compresses literals and
|
|
/// sequences into one compressed-block body.
|
|
#[allow(clippy::too_many_arguments)]
|
|
unsafe fn entropy_compress_seq_store_internal(
|
|
dst: *mut c_void,
|
|
dst_capacity: usize,
|
|
literals: *const c_void,
|
|
lit_size: usize,
|
|
seq_store: *const SeqStore_t,
|
|
prev_entropy: *const ZSTD_entropyCTables_t,
|
|
next_entropy: *mut ZSTD_entropyCTables_t,
|
|
strategy: c_int,
|
|
disable_literal_compression: c_int,
|
|
entropy_workspace: *mut c_void,
|
|
entropy_wksp_size: usize,
|
|
bmi2: c_int,
|
|
) -> usize {
|
|
let count = entropy_workspace.cast::<c_uint>();
|
|
let entropy_workspace = unsafe { count.add(MAX_SEQ + 1) }.cast::<c_void>();
|
|
let entropy_wksp_size = entropy_wksp_size.wrapping_sub((MAX_SEQ + 1) * size_of::<c_uint>());
|
|
let store = unsafe { &*seq_store };
|
|
let sequences = store.sequencesStart;
|
|
let nb_seq = unsafe { store.sequences.offset_from(sequences) } as usize;
|
|
let ostart = dst.cast::<u8>();
|
|
let mut written = 0usize;
|
|
let last_count_size;
|
|
let long_offsets;
|
|
|
|
/* Compress literals */
|
|
{
|
|
/* Base suspicion of uncompressibility on ratio of literals to sequences */
|
|
let suspect_uncompressible =
|
|
c_int::from(nb_seq == 0 || lit_size / nb_seq >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO);
|
|
let c_size = unsafe {
|
|
ZSTD_compressLiterals(
|
|
dst,
|
|
dst_capacity,
|
|
literals,
|
|
lit_size,
|
|
entropy_workspace,
|
|
entropy_wksp_size,
|
|
ptr::addr_of!((*prev_entropy).huf),
|
|
ptr::addr_of_mut!((*next_entropy).huf),
|
|
strategy,
|
|
disable_literal_compression,
|
|
suspect_uncompressible,
|
|
bmi2,
|
|
)
|
|
};
|
|
if ERR_isError(c_size) {
|
|
return c_size;
|
|
}
|
|
debug_assert!(c_size <= dst_capacity);
|
|
written += c_size;
|
|
}
|
|
|
|
/* Sequences Header */
|
|
if (dst_capacity as isize).wrapping_sub(written as isize) < 3 /* max nbSeq Size */ + 1
|
|
/* seqHead */
|
|
{
|
|
return ERROR(ZstdErrorCode::DstSizeTooSmall);
|
|
}
|
|
unsafe {
|
|
if nb_seq < 128 {
|
|
*ostart.add(written) = nb_seq as u8;
|
|
written += 1;
|
|
} else if nb_seq < LONGNBSEQ as usize {
|
|
*ostart.add(written) = ((nb_seq >> 8) + 0x80) as u8;
|
|
*ostart.add(written + 1) = nb_seq as u8;
|
|
written += 2;
|
|
} else {
|
|
*ostart.add(written) = 0xFF;
|
|
MEM_writeLE16(
|
|
ostart.add(written + 1).cast::<c_void>(),
|
|
(nb_seq - LONGNBSEQ as usize) as u16,
|
|
);
|
|
written += 3;
|
|
}
|
|
}
|
|
if nb_seq == 0 {
|
|
/* Copy the old tables over as if we repeated them */
|
|
unsafe {
|
|
copy_fse_tables(
|
|
ptr::addr_of_mut!((*next_entropy).fse),
|
|
ptr::addr_of!((*prev_entropy).fse),
|
|
)
|
|
};
|
|
return written;
|
|
}
|
|
{
|
|
let seq_head = written;
|
|
written += 1;
|
|
/* build stats for sequences */
|
|
let stats = unsafe {
|
|
build_sequences_statistics(
|
|
seq_store,
|
|
nb_seq,
|
|
ptr::addr_of!((*prev_entropy).fse),
|
|
ptr::addr_of_mut!((*next_entropy).fse),
|
|
ostart.add(written),
|
|
dst_capacity - written,
|
|
strategy,
|
|
count,
|
|
entropy_workspace,
|
|
entropy_wksp_size,
|
|
)
|
|
};
|
|
if ERR_isError(stats.size) {
|
|
return stats.size;
|
|
}
|
|
unsafe {
|
|
*ostart.add(seq_head) =
|
|
((stats.ll_type << 6) + (stats.off_type << 4) + (stats.ml_type << 2)) as u8;
|
|
}
|
|
last_count_size = stats.last_count_size;
|
|
written += stats.size;
|
|
long_offsets = stats.long_offsets;
|
|
}
|
|
|
|
{
|
|
let bitstream_size = unsafe {
|
|
ZSTD_encodeSequences(
|
|
ostart.add(written).cast::<c_void>(),
|
|
dst_capacity - written,
|
|
ptr::addr_of!((*next_entropy).fse.matchlengthCTable).cast::<u32>(),
|
|
store.mlCode,
|
|
ptr::addr_of!((*next_entropy).fse.offcodeCTable).cast::<u32>(),
|
|
store.ofCode,
|
|
ptr::addr_of!((*next_entropy).fse.litlengthCTable).cast::<u32>(),
|
|
store.llCode,
|
|
sequences,
|
|
nb_seq,
|
|
long_offsets,
|
|
bmi2,
|
|
)
|
|
};
|
|
if ERR_isError(bitstream_size) {
|
|
return bitstream_size;
|
|
}
|
|
written += bitstream_size;
|
|
/* zstd versions <= 1.3.4 mistakenly report corruption when
|
|
* FSE_readNCount() receives a buffer < 4 bytes.
|
|
* Fixed by https://github.com/facebook/zstd/pull/1146.
|
|
* This can happen when the last set_compressed table present is 2
|
|
* bytes and the bitstream is only one byte.
|
|
* In this exceedingly rare case, we will simply emit an uncompressed
|
|
* block, since it isn't worth optimizing.
|
|
*/
|
|
if last_count_size != 0 && last_count_size + bitstream_size < 4 {
|
|
/* last_count_size >= 2 && bitstream_size > 0 ==> last_count_size == 3 */
|
|
debug_assert!(last_count_size + bitstream_size == 3);
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
written
|
|
}
|
|
|
|
/// C's `ZSTD_entropyCompressSeqStore_wExtLitBuffer()`: adds the
|
|
/// incompressibility fallbacks around the internal entropy pass.
|
|
#[allow(clippy::too_many_arguments)]
|
|
unsafe fn entropy_compress_seq_store_w_ext_lit_buffer(
|
|
dst: *mut c_void,
|
|
dst_capacity: usize,
|
|
literals: *const c_void,
|
|
lit_size: usize,
|
|
block_size: usize,
|
|
seq_store: *const SeqStore_t,
|
|
prev_entropy: *const ZSTD_entropyCTables_t,
|
|
next_entropy: *mut ZSTD_entropyCTables_t,
|
|
strategy: c_int,
|
|
disable_literal_compression: c_int,
|
|
entropy_workspace: *mut c_void,
|
|
entropy_wksp_size: usize,
|
|
bmi2: c_int,
|
|
) -> usize {
|
|
let c_size = unsafe {
|
|
entropy_compress_seq_store_internal(
|
|
dst,
|
|
dst_capacity,
|
|
literals,
|
|
lit_size,
|
|
seq_store,
|
|
prev_entropy,
|
|
next_entropy,
|
|
strategy,
|
|
disable_literal_compression,
|
|
entropy_workspace,
|
|
entropy_wksp_size,
|
|
bmi2,
|
|
)
|
|
};
|
|
if c_size == 0 {
|
|
return 0;
|
|
}
|
|
/* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block.
|
|
* Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block.
|
|
*/
|
|
if c_size == ERROR(ZstdErrorCode::DstSizeTooSmall) && block_size <= dst_capacity {
|
|
return 0; /* block not compressed */
|
|
}
|
|
if ERR_isError(c_size) {
|
|
return c_size;
|
|
}
|
|
|
|
/* Check compressibility */
|
|
let max_c_size = block_size.wrapping_sub(min_gain(block_size, strategy));
|
|
if c_size >= max_c_size {
|
|
return 0; /* block not compressed */
|
|
}
|
|
c_size
|
|
}
|
|
|
|
/// C ABI wrapper for the external-literal-buffer entropy pass; the C shim
|
|
/// keeps the `ZSTD_entropyCompressSeqStore_internal()` name and extracts the
|
|
/// two `ZSTD_CCtx_params` scalars.
|
|
#[allow(clippy::too_many_arguments)]
|
|
#[no_mangle]
|
|
pub unsafe extern "C" fn ZSTD_rust_entropyCompressSeqStore_internal(
|
|
dst: *mut c_void,
|
|
dst_capacity: usize,
|
|
literals: *const c_void,
|
|
lit_size: usize,
|
|
seq_store: *const SeqStore_t,
|
|
prev_entropy: *const ZSTD_entropyCTables_t,
|
|
next_entropy: *mut ZSTD_entropyCTables_t,
|
|
strategy: c_int,
|
|
disable_literal_compression: c_int,
|
|
entropy_workspace: *mut c_void,
|
|
entropy_wksp_size: usize,
|
|
bmi2: c_int,
|
|
) -> usize {
|
|
unsafe {
|
|
entropy_compress_seq_store_internal(
|
|
dst,
|
|
dst_capacity,
|
|
literals,
|
|
lit_size,
|
|
seq_store,
|
|
prev_entropy,
|
|
next_entropy,
|
|
strategy,
|
|
disable_literal_compression,
|
|
entropy_workspace,
|
|
entropy_wksp_size,
|
|
bmi2,
|
|
)
|
|
}
|
|
}
|
|
|
|
/// C ABI implementation behind the `ZSTD_entropyCompressSeqStore()` shim.
|
|
#[allow(clippy::too_many_arguments)]
|
|
#[no_mangle]
|
|
pub unsafe extern "C" fn ZSTD_rust_entropyCompressSeqStore(
|
|
seq_store: *const SeqStore_t,
|
|
prev_entropy: *const ZSTD_entropyCTables_t,
|
|
next_entropy: *mut ZSTD_entropyCTables_t,
|
|
strategy: c_int,
|
|
disable_literal_compression: c_int,
|
|
dst: *mut c_void,
|
|
dst_capacity: usize,
|
|
src_size: usize,
|
|
entropy_workspace: *mut c_void,
|
|
entropy_wksp_size: usize,
|
|
bmi2: c_int,
|
|
) -> usize {
|
|
let store = unsafe { &*seq_store };
|
|
let lit_size = unsafe { store.lit.offset_from(store.litStart) } as usize;
|
|
unsafe {
|
|
entropy_compress_seq_store_w_ext_lit_buffer(
|
|
dst,
|
|
dst_capacity,
|
|
store.litStart.cast::<c_void>(),
|
|
lit_size,
|
|
src_size,
|
|
seq_store,
|
|
prev_entropy,
|
|
next_entropy,
|
|
strategy,
|
|
disable_literal_compression,
|
|
entropy_workspace,
|
|
entropy_wksp_size,
|
|
bmi2,
|
|
)
|
|
}
|
|
}
|
|
|
|
/// C's `ZSTD_buildBlockEntropyStats_literals()`.
|
|
///
|
|
/// Stores the literals block type and Huffman description table into
|
|
/// `huf_metadata` and returns the description size, or an error code.
|
|
#[allow(clippy::too_many_arguments)]
|
|
unsafe fn build_block_entropy_stats_literals(
|
|
src: *const c_void,
|
|
src_size: usize,
|
|
prev_huf: *const ZSTD_hufCTables_t,
|
|
next_huf: *mut ZSTD_hufCTables_t,
|
|
huf_metadata: *mut ZSTD_hufCTablesMetadata_t,
|
|
literals_compression_is_disabled: c_int,
|
|
workspace: *mut c_void,
|
|
wksp_size: usize,
|
|
huf_flags: c_int,
|
|
) -> usize {
|
|
let count_wksp = workspace.cast::<c_uint>();
|
|
let count_wksp_size = (HUF_SYMBOLVALUE_MAX as usize + 1) * size_of::<c_uint>();
|
|
let node_wksp = unsafe { workspace.cast::<u8>().add(count_wksp_size) }.cast::<c_void>();
|
|
let node_wksp_size = wksp_size.wrapping_sub(count_wksp_size);
|
|
let mut max_symbol_value = HUF_SYMBOLVALUE_MAX;
|
|
let mut huff_log = LIT_HUF_LOG;
|
|
let mut repeat = unsafe { (*prev_huf).repeatMode };
|
|
|
|
/* Prepare nextEntropy assuming reusing the existing table */
|
|
unsafe { copy_huf_tables(next_huf, prev_huf) };
|
|
|
|
if literals_compression_is_disabled != 0 {
|
|
unsafe { (*huf_metadata).hType = SET_BASIC };
|
|
return 0;
|
|
}
|
|
|
|
/* small ? don't even attempt compression (speed opt) */
|
|
{
|
|
let min_lit_size = if repeat == HUF_REPEAT_VALID {
|
|
6
|
|
} else {
|
|
COMPRESS_LITERALS_SIZE_MIN
|
|
};
|
|
if src_size <= min_lit_size {
|
|
unsafe { (*huf_metadata).hType = SET_BASIC };
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
/* Scan input and build symbol stats */
|
|
{
|
|
let largest = unsafe {
|
|
HIST_count_wksp(
|
|
count_wksp,
|
|
&mut max_symbol_value,
|
|
src,
|
|
src_size,
|
|
workspace,
|
|
wksp_size,
|
|
)
|
|
};
|
|
if ERR_isError(largest) {
|
|
return largest;
|
|
}
|
|
if largest == src_size {
|
|
/* only one literal symbol */
|
|
unsafe { (*huf_metadata).hType = SET_RLE };
|
|
return 0;
|
|
}
|
|
if largest <= (src_size >> 7) + 4 {
|
|
/* heuristic: likely not compressible */
|
|
unsafe { (*huf_metadata).hType = SET_BASIC };
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
/* Validate the previous Huffman table */
|
|
if repeat == HUF_REPEAT_CHECK
|
|
&& unsafe {
|
|
HUF_validateCTable(
|
|
ptr::addr_of!((*prev_huf).CTable).cast::<usize>(),
|
|
count_wksp,
|
|
max_symbol_value,
|
|
)
|
|
} == 0
|
|
{
|
|
repeat = HUF_REPEAT_NONE;
|
|
}
|
|
|
|
/* Build Huffman Tree */
|
|
unsafe {
|
|
ptr::write_bytes(
|
|
ptr::addr_of_mut!((*next_huf).CTable).cast::<u8>(),
|
|
0,
|
|
size_of_val(&(*next_huf).CTable),
|
|
);
|
|
}
|
|
huff_log = unsafe {
|
|
HUF_optimalTableLog(
|
|
huff_log,
|
|
src_size,
|
|
max_symbol_value,
|
|
node_wksp,
|
|
node_wksp_size,
|
|
ptr::addr_of_mut!((*next_huf).CTable).cast::<usize>(),
|
|
count_wksp,
|
|
huf_flags,
|
|
)
|
|
};
|
|
debug_assert!(huff_log <= LIT_HUF_LOG);
|
|
{
|
|
let max_bits = unsafe {
|
|
HUF_buildCTable_wksp(
|
|
ptr::addr_of_mut!((*next_huf).CTable).cast::<usize>(),
|
|
count_wksp,
|
|
max_symbol_value,
|
|
huff_log,
|
|
node_wksp,
|
|
node_wksp_size,
|
|
)
|
|
};
|
|
if ERR_isError(max_bits) {
|
|
return max_bits;
|
|
}
|
|
huff_log = max_bits as c_uint;
|
|
}
|
|
{
|
|
/* Build and write the CTable */
|
|
let new_c_size = unsafe {
|
|
HUF_estimateCompressedSize(
|
|
ptr::addr_of!((*next_huf).CTable).cast::<usize>(),
|
|
count_wksp,
|
|
max_symbol_value,
|
|
)
|
|
};
|
|
let h_size = unsafe {
|
|
HUF_writeCTable_wksp(
|
|
ptr::addr_of_mut!((*huf_metadata).hufDesBuffer).cast::<c_void>(),
|
|
size_of::<[u8; ZSTD_MAX_HUF_HEADER_SIZE]>(),
|
|
ptr::addr_of!((*next_huf).CTable).cast::<usize>(),
|
|
max_symbol_value,
|
|
huff_log,
|
|
node_wksp,
|
|
node_wksp_size,
|
|
)
|
|
};
|
|
/* Check against repeating the previous CTable */
|
|
if repeat != HUF_REPEAT_NONE {
|
|
let old_c_size = unsafe {
|
|
HUF_estimateCompressedSize(
|
|
ptr::addr_of!((*prev_huf).CTable).cast::<usize>(),
|
|
count_wksp,
|
|
max_symbol_value,
|
|
)
|
|
};
|
|
if old_c_size < src_size
|
|
&& (old_c_size <= h_size.wrapping_add(new_c_size)
|
|
|| h_size.wrapping_add(12) >= src_size)
|
|
{
|
|
unsafe {
|
|
copy_huf_tables(next_huf, prev_huf);
|
|
(*huf_metadata).hType = SET_REPEAT;
|
|
}
|
|
return 0;
|
|
}
|
|
}
|
|
if new_c_size.wrapping_add(h_size) >= src_size {
|
|
unsafe {
|
|
copy_huf_tables(next_huf, prev_huf);
|
|
(*huf_metadata).hType = SET_BASIC;
|
|
}
|
|
return 0;
|
|
}
|
|
unsafe {
|
|
(*huf_metadata).hType = SET_COMPRESSED;
|
|
(*next_huf).repeatMode = HUF_REPEAT_CHECK;
|
|
}
|
|
h_size
|
|
}
|
|
}
|
|
|
|
/// C's `ZSTD_buildDummySequencesStatistics()`: marks every encoding type as
|
|
/// basic and resets the FSE repeat modes.
|
|
unsafe fn build_dummy_sequences_statistics(
|
|
next_entropy: *mut ZSTD_fseCTables_t,
|
|
) -> SymbolEncodingTypeStats {
|
|
unsafe {
|
|
(*next_entropy).litlength_repeatMode = FSE_REPEAT_NONE;
|
|
(*next_entropy).offcode_repeatMode = FSE_REPEAT_NONE;
|
|
(*next_entropy).matchlength_repeatMode = FSE_REPEAT_NONE;
|
|
}
|
|
SymbolEncodingTypeStats {
|
|
ll_type: SET_BASIC as u32,
|
|
off_type: SET_BASIC as u32,
|
|
ml_type: SET_BASIC as u32,
|
|
size: 0,
|
|
last_count_size: 0,
|
|
long_offsets: 0,
|
|
}
|
|
}
|
|
|
|
/// C's `ZSTD_buildBlockEntropyStats_sequences()`.
|
|
#[allow(clippy::too_many_arguments)]
|
|
unsafe fn build_block_entropy_stats_sequences(
|
|
seq_store: *const SeqStore_t,
|
|
prev_entropy: *const ZSTD_fseCTables_t,
|
|
next_entropy: *mut ZSTD_fseCTables_t,
|
|
strategy: c_int,
|
|
fse_metadata: *mut ZSTD_fseCTablesMetadata_t,
|
|
workspace: *mut c_void,
|
|
wksp_size: usize,
|
|
) -> usize {
|
|
let store = unsafe { &*seq_store };
|
|
let nb_seq = unsafe { store.sequences.offset_from(store.sequencesStart) } as usize;
|
|
let count_workspace = workspace.cast::<c_uint>();
|
|
let entropy_workspace = unsafe { count_workspace.add(MAX_SEQ + 1) }.cast::<c_void>();
|
|
let entropy_workspace_size = wksp_size.wrapping_sub((MAX_SEQ + 1) * size_of::<c_uint>());
|
|
|
|
let stats = if nb_seq != 0 {
|
|
unsafe {
|
|
build_sequences_statistics(
|
|
seq_store,
|
|
nb_seq,
|
|
prev_entropy,
|
|
next_entropy,
|
|
ptr::addr_of_mut!((*fse_metadata).fseTablesBuffer).cast::<u8>(),
|
|
ZSTD_MAX_FSE_HEADERS_SIZE,
|
|
strategy,
|
|
count_workspace,
|
|
entropy_workspace,
|
|
entropy_workspace_size,
|
|
)
|
|
}
|
|
} else {
|
|
unsafe { build_dummy_sequences_statistics(next_entropy) }
|
|
};
|
|
if ERR_isError(stats.size) {
|
|
return stats.size;
|
|
}
|
|
unsafe {
|
|
(*fse_metadata).llType = stats.ll_type as c_int;
|
|
(*fse_metadata).ofType = stats.off_type as c_int;
|
|
(*fse_metadata).mlType = stats.ml_type as c_int;
|
|
(*fse_metadata).lastCountSize = stats.last_count_size;
|
|
}
|
|
stats.size
|
|
}
|
|
|
|
/// C's `ZSTD_buildBlockEntropyStats()`: builds entropy statistics for one
|
|
/// block. Requires an `ENTROPY_WORKSPACE_SIZE` workspace.
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub(crate) unsafe fn build_block_entropy_stats(
|
|
seq_store: *const SeqStore_t,
|
|
prev_entropy: *const ZSTD_entropyCTables_t,
|
|
next_entropy: *mut ZSTD_entropyCTables_t,
|
|
strategy: c_int,
|
|
disable_literal_compression: c_int,
|
|
entropy_metadata: *mut ZSTD_entropyCTablesMetadata_t,
|
|
workspace: *mut c_void,
|
|
wksp_size: usize,
|
|
) -> usize {
|
|
let store = unsafe { &*seq_store };
|
|
let lit_size = unsafe { store.lit.offset_from(store.litStart) } as usize;
|
|
let huf_flags = if strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD {
|
|
HUF_FLAGS_OPTIMAL_DEPTH
|
|
} else {
|
|
0
|
|
};
|
|
|
|
let huf_des_size = unsafe {
|
|
build_block_entropy_stats_literals(
|
|
store.litStart.cast::<c_void>(),
|
|
lit_size,
|
|
ptr::addr_of!((*prev_entropy).huf),
|
|
ptr::addr_of_mut!((*next_entropy).huf),
|
|
ptr::addr_of_mut!((*entropy_metadata).hufMetadata),
|
|
disable_literal_compression,
|
|
workspace,
|
|
wksp_size,
|
|
huf_flags,
|
|
)
|
|
};
|
|
unsafe { (*entropy_metadata).hufMetadata.hufDesSize = huf_des_size };
|
|
if ERR_isError(huf_des_size) {
|
|
return huf_des_size;
|
|
}
|
|
|
|
let fse_tables_size = unsafe {
|
|
build_block_entropy_stats_sequences(
|
|
seq_store,
|
|
ptr::addr_of!((*prev_entropy).fse),
|
|
ptr::addr_of_mut!((*next_entropy).fse),
|
|
strategy,
|
|
ptr::addr_of_mut!((*entropy_metadata).fseMetadata),
|
|
workspace,
|
|
wksp_size,
|
|
)
|
|
};
|
|
unsafe { (*entropy_metadata).fseMetadata.fseTablesSize = fse_tables_size };
|
|
if ERR_isError(fse_tables_size) {
|
|
return fse_tables_size;
|
|
}
|
|
0
|
|
}
|
|
|
|
/// C ABI wrapper behind the `ZSTD_buildBlockEntropyStats()` shim, which is
|
|
/// still exported from C for the block splitter's size estimation.
|
|
#[allow(clippy::too_many_arguments)]
|
|
#[no_mangle]
|
|
pub unsafe extern "C" fn ZSTD_rust_buildBlockEntropyStats(
|
|
seq_store: *const SeqStore_t,
|
|
prev_entropy: *const ZSTD_entropyCTables_t,
|
|
next_entropy: *mut ZSTD_entropyCTables_t,
|
|
strategy: c_int,
|
|
disable_literal_compression: c_int,
|
|
entropy_metadata: *mut ZSTD_entropyCTablesMetadata_t,
|
|
workspace: *mut c_void,
|
|
wksp_size: usize,
|
|
) -> usize {
|
|
unsafe {
|
|
build_block_entropy_stats(
|
|
seq_store,
|
|
prev_entropy,
|
|
next_entropy,
|
|
strategy,
|
|
disable_literal_compression,
|
|
entropy_metadata,
|
|
workspace,
|
|
wksp_size,
|
|
)
|
|
}
|
|
}
|
|
|
|
/// C's `ZSTD_copyBlockSequences()`: exports a block's sequences in the public
|
|
/// `ZSTD_Sequence` format, appending the last-literals block delimiter.
|
|
#[no_mangle]
|
|
pub unsafe extern "C" fn ZSTD_rust_copyBlockSequences(
|
|
seq_collector: *mut SeqCollector,
|
|
seq_store: *const SeqStore_t,
|
|
prev_repcodes: *const u32,
|
|
) -> usize {
|
|
let collector = unsafe { &mut *seq_collector };
|
|
let store = unsafe { &*seq_store };
|
|
let in_seqs = store.sequencesStart;
|
|
let nb_in_sequences = unsafe { store.sequences.offset_from(in_seqs) } as usize;
|
|
let nb_in_literals = unsafe { store.lit.offset_from(store.litStart) } as usize;
|
|
let out_seqs = if collector.seqIndex == 0 {
|
|
collector.seqStart
|
|
} else {
|
|
unsafe { collector.seqStart.add(collector.seqIndex) }
|
|
};
|
|
let nb_out_sequences = nb_in_sequences + 1;
|
|
let mut nb_out_literals = 0usize;
|
|
|
|
/* Bounds check that we have enough space for every input sequence
|
|
* and the block delimiter
|
|
*/
|
|
debug_assert!(collector.seqIndex <= collector.maxSequences);
|
|
if nb_out_sequences > collector.maxSequences.wrapping_sub(collector.seqIndex) {
|
|
return ERROR(ZstdErrorCode::DstSizeTooSmall);
|
|
}
|
|
|
|
let mut repcodes = [0u32; ZSTD_REP_NUM];
|
|
unsafe { ptr::copy_nonoverlapping(prev_repcodes, repcodes.as_mut_ptr(), ZSTD_REP_NUM) };
|
|
for index in 0..nb_in_sequences {
|
|
let in_seq = unsafe { in_seqs.add(index).read() };
|
|
let out = unsafe { out_seqs.add(index) };
|
|
let mut lit_length = in_seq.litLength as u32;
|
|
let mut match_length = in_seq.mlBase as u32 + MINMATCH as u32;
|
|
let mut rep = 0u32;
|
|
|
|
/* Handle the possible single length >= 64K
|
|
* There can only be one because we add MINMATCH to every match length,
|
|
* and blocks are at most 128K.
|
|
*/
|
|
if index == store.longLengthPos as usize {
|
|
if store.longLengthType == ZSTD_LLT_LITERAL_LENGTH {
|
|
lit_length += 0x10000;
|
|
} else if store.longLengthType == ZSTD_LLT_MATCH_LENGTH {
|
|
match_length += 0x10000;
|
|
}
|
|
}
|
|
|
|
/* Determine the raw offset given the offBase, which may be a repcode. */
|
|
let raw_offset = if (1..=ZSTD_REP_NUM as u32).contains(&in_seq.offBase) {
|
|
let repcode = in_seq.offBase;
|
|
debug_assert!(repcode > 0);
|
|
rep = repcode;
|
|
if lit_length != 0 {
|
|
repcodes[repcode as usize - 1]
|
|
} else if repcode == 3 {
|
|
debug_assert!(repcodes[0] > 1);
|
|
repcodes[0].wrapping_sub(1)
|
|
} else {
|
|
repcodes[repcode as usize]
|
|
}
|
|
} else {
|
|
in_seq.offBase.wrapping_sub(ZSTD_REP_NUM as u32)
|
|
};
|
|
unsafe {
|
|
(*out).litLength = lit_length;
|
|
(*out).matchLength = match_length;
|
|
(*out).offset = raw_offset;
|
|
(*out).rep = rep;
|
|
}
|
|
|
|
/* Update repcode history for the sequence */
|
|
update_rep(&mut repcodes, in_seq.offBase, in_seq.litLength == 0);
|
|
|
|
nb_out_literals += lit_length as usize;
|
|
}
|
|
/* Insert last literals (if any exist) in the block as a sequence with ml == off == 0.
|
|
* If there are no last literals, then we'll emit (of: 0, ml: 0, ll: 0), which is a marker
|
|
* for the block boundary, according to the API.
|
|
*/
|
|
debug_assert!(nb_in_literals >= nb_out_literals);
|
|
{
|
|
let last_ll_size = nb_in_literals.wrapping_sub(nb_out_literals);
|
|
unsafe {
|
|
let last = out_seqs.add(nb_in_sequences);
|
|
(*last).litLength = last_ll_size as u32;
|
|
(*last).matchLength = 0;
|
|
(*last).offset = 0;
|
|
}
|
|
}
|
|
collector.seqIndex += nb_out_sequences;
|
|
debug_assert!(collector.seqIndex <= collector.maxSequences);
|
|
|
|
0
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use std::mem::{align_of, offset_of};
|
|
|
|
const HUF_CTABLE_SIZE_ST: usize = HUF_SYMBOLVALUE_MAX as usize + 2;
|
|
const ENTROPY_WORKSPACE_SIZE: usize = (8 << 10) + 512 + (MAX_SEQ + 2) * size_of::<u32>();
|
|
|
|
fn empty_huf_tables(repeat: c_int) -> ZSTD_hufCTables_t {
|
|
ZSTD_hufCTables_t {
|
|
CTable: [0; HUF_CTABLE_SIZE_ST],
|
|
repeatMode: repeat,
|
|
}
|
|
}
|
|
|
|
fn empty_fse_tables(repeat: c_int) -> ZSTD_fseCTables_t {
|
|
ZSTD_fseCTables_t {
|
|
offcodeCTable: [0; OFF_CTABLE_SIZE],
|
|
matchlengthCTable: [0; ML_CTABLE_SIZE],
|
|
litlengthCTable: [0; LL_CTABLE_SIZE],
|
|
offcode_repeatMode: repeat,
|
|
matchlength_repeatMode: repeat,
|
|
litlength_repeatMode: repeat,
|
|
}
|
|
}
|
|
|
|
struct TestSeqStore {
|
|
sequences: Vec<SeqDef>,
|
|
literals: Vec<u8>,
|
|
ll_codes: Vec<u8>,
|
|
ml_codes: Vec<u8>,
|
|
of_codes: Vec<u8>,
|
|
}
|
|
|
|
impl TestSeqStore {
|
|
fn new(sequences: Vec<SeqDef>, literals: Vec<u8>) -> Self {
|
|
let nb_seq = sequences.len();
|
|
TestSeqStore {
|
|
sequences,
|
|
literals,
|
|
ll_codes: vec![0; nb_seq],
|
|
ml_codes: vec![0; nb_seq],
|
|
of_codes: vec![0; nb_seq],
|
|
}
|
|
}
|
|
|
|
fn seq_store(&mut self) -> SeqStore_t {
|
|
let nb_seq = self.sequences.len();
|
|
let nb_lit = self.literals.len();
|
|
SeqStore_t {
|
|
sequencesStart: self.sequences.as_mut_ptr(),
|
|
sequences: unsafe { self.sequences.as_mut_ptr().add(nb_seq) },
|
|
litStart: self.literals.as_mut_ptr(),
|
|
lit: unsafe { self.literals.as_mut_ptr().add(nb_lit) },
|
|
llCode: self.ll_codes.as_mut_ptr(),
|
|
mlCode: self.ml_codes.as_mut_ptr(),
|
|
ofCode: self.of_codes.as_mut_ptr(),
|
|
maxNbSeq: nb_seq,
|
|
maxNbLit: nb_lit,
|
|
longLengthType: 0,
|
|
longLengthPos: 0,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn c_leaf_layouts_match_supported_abis() {
|
|
assert_eq!(size_of::<SeqDef>(), 8);
|
|
assert_eq!(align_of::<SeqDef>(), align_of::<u32>());
|
|
assert_eq!(offset_of!(SeqStore_t, sequencesStart), 0);
|
|
assert_eq!(
|
|
offset_of!(SeqStore_t, longLengthPos),
|
|
9 * size_of::<usize>() + 4
|
|
);
|
|
assert_eq!(size_of::<SeqStore_t>(), 9 * size_of::<usize>() + 8);
|
|
assert_eq!(size_of::<ZSTD_fseCTables_t>(), 3552);
|
|
assert_eq!(offset_of!(ZSTD_compressedBlockState_t, entropy), 0);
|
|
assert_eq!(
|
|
offset_of!(ZSTD_compressedBlockState_t, rep),
|
|
size_of::<ZSTD_entropyCTables_t>()
|
|
);
|
|
assert_eq!(size_of::<ZSTD_Sequence>(), 16);
|
|
assert_eq!(offset_of!(SeqCollector, seqStart), size_of::<usize>());
|
|
assert_eq!(offset_of!(SeqCollector, seqIndex), 2 * size_of::<usize>());
|
|
assert_eq!(size_of::<SeqCollector>(), 4 * size_of::<usize>());
|
|
if size_of::<usize>() == 8 {
|
|
assert_eq!(size_of::<ZSTD_hufCTablesMetadata_t>(), 144);
|
|
assert_eq!(size_of::<ZSTD_fseCTablesMetadata_t>(), 168);
|
|
assert_eq!(size_of::<ZSTD_entropyCTablesMetadata_t>(), 312);
|
|
assert_eq!(size_of::<ZSTD_entropyCTables_t>(), 5616);
|
|
assert_eq!(size_of::<ZSTD_compressedBlockState_t>(), 5632);
|
|
} else {
|
|
assert_eq!(size_of::<ZSTD_hufCTablesMetadata_t>(), 136);
|
|
assert_eq!(size_of::<ZSTD_fseCTablesMetadata_t>(), 156);
|
|
assert_eq!(size_of::<ZSTD_entropyCTablesMetadata_t>(), 292);
|
|
assert_eq!(size_of::<ZSTD_entropyCTables_t>(), 4584);
|
|
assert_eq!(size_of::<ZSTD_compressedBlockState_t>(), 4596);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn compressed_block_state_reset_restores_repcodes_and_repeat_modes() {
|
|
let mut block_state =
|
|
unsafe { std::mem::MaybeUninit::<ZSTD_compressedBlockState_t>::zeroed().assume_init() };
|
|
block_state.rep = [99; ZSTD_REP_NUM];
|
|
block_state.entropy.huf.repeatMode = HUF_REPEAT_VALID;
|
|
block_state.entropy.fse.offcode_repeatMode = 2;
|
|
block_state.entropy.fse.matchlength_repeatMode = 1;
|
|
block_state.entropy.fse.litlength_repeatMode = 2;
|
|
|
|
unsafe { ZSTD_rust_resetCompressedBlockState(&mut block_state) };
|
|
|
|
assert_eq!(block_state.rep, [1, 4, 8]);
|
|
assert_eq!(block_state.entropy.huf.repeatMode, HUF_REPEAT_NONE);
|
|
assert_eq!(block_state.entropy.fse.offcode_repeatMode, FSE_REPEAT_NONE);
|
|
assert_eq!(
|
|
block_state.entropy.fse.matchlength_repeatMode,
|
|
FSE_REPEAT_NONE
|
|
);
|
|
assert_eq!(
|
|
block_state.entropy.fse.litlength_repeatMode,
|
|
FSE_REPEAT_NONE
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn store_last_literals_copies_bytes_and_advances_only_the_literal_cursor() {
|
|
let mut sequences = [SeqDef::default(); 2];
|
|
let mut literals = [0xA5u8; 8];
|
|
let sequences_start = sequences.as_mut_ptr();
|
|
let literals_start = literals.as_mut_ptr();
|
|
let mut seq_store = SeqStore_t {
|
|
sequencesStart: sequences_start,
|
|
sequences: unsafe { sequences_start.add(1) },
|
|
litStart: literals_start,
|
|
lit: unsafe { literals_start.add(2) },
|
|
llCode: std::ptr::null_mut(),
|
|
mlCode: std::ptr::null_mut(),
|
|
ofCode: std::ptr::null_mut(),
|
|
maxNbSeq: sequences.len(),
|
|
maxNbLit: literals.len(),
|
|
longLengthType: ZSTD_LLT_MATCH_LENGTH,
|
|
longLengthPos: 7,
|
|
};
|
|
let source = [1u8, 2, 3, 4];
|
|
|
|
unsafe {
|
|
ZSTD_rust_storeLastLiterals(&mut seq_store, source.as_ptr(), 3);
|
|
}
|
|
assert_eq!(literals, [0xA5, 0xA5, 1, 2, 3, 0xA5, 0xA5, 0xA5]);
|
|
assert_eq!(seq_store.lit, unsafe { literals_start.add(5) });
|
|
assert_eq!(seq_store.sequences, unsafe { sequences_start.add(1) });
|
|
|
|
let cursor = seq_store.lit;
|
|
unsafe {
|
|
ZSTD_rust_storeLastLiterals(&mut seq_store, source.as_ptr(), 0);
|
|
}
|
|
assert_eq!(seq_store.lit, cursor);
|
|
assert_eq!(literals, [0xA5, 0xA5, 1, 2, 3, 0xA5, 0xA5, 0xA5]);
|
|
}
|
|
|
|
#[test]
|
|
fn reset_seq_store_resets_cursors_and_long_length_type_but_not_position() {
|
|
let mut sequences = [SeqDef::default(); 2];
|
|
let mut literals = [0u8; 5];
|
|
let sequences_start = sequences.as_mut_ptr();
|
|
let literals_start = literals.as_mut_ptr();
|
|
let mut seq_store = SeqStore_t {
|
|
sequencesStart: sequences_start,
|
|
sequences: unsafe { sequences_start.add(2) },
|
|
litStart: literals_start,
|
|
lit: unsafe { literals_start.add(4) },
|
|
llCode: std::ptr::null_mut(),
|
|
mlCode: std::ptr::null_mut(),
|
|
ofCode: std::ptr::null_mut(),
|
|
maxNbSeq: sequences.len(),
|
|
maxNbLit: literals.len(),
|
|
longLengthType: ZSTD_LLT_LITERAL_LENGTH,
|
|
longLengthPos: 11,
|
|
};
|
|
|
|
unsafe { ZSTD_rust_resetSeqStore(&mut seq_store) };
|
|
|
|
assert_eq!(seq_store.sequences, sequences_start);
|
|
assert_eq!(seq_store.lit, literals_start);
|
|
assert_eq!(seq_store.longLengthType, 0);
|
|
assert_eq!(seq_store.longLengthPos, 11);
|
|
}
|
|
|
|
#[test]
|
|
fn fast_sequence_length_sum_handles_empty_and_populated_buffers() {
|
|
assert_eq!(
|
|
unsafe { ZSTD_rust_fastSequenceLengthSum(std::ptr::null(), 0) },
|
|
0
|
|
);
|
|
|
|
let sequences = [
|
|
ZSTD_Sequence {
|
|
offset: 0,
|
|
litLength: 3,
|
|
matchLength: 7,
|
|
rep: 0,
|
|
},
|
|
ZSTD_Sequence {
|
|
offset: 10,
|
|
litLength: 11,
|
|
matchLength: 13,
|
|
rep: 1,
|
|
},
|
|
];
|
|
assert_eq!(
|
|
unsafe { ZSTD_rust_fastSequenceLengthSum(sequences.as_ptr(), sequences.len()) },
|
|
34
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn is_rle_matches_empty_one_byte_and_unrolled_inputs() {
|
|
let one_byte = [0x5Au8];
|
|
assert_eq!(unsafe { ZSTD_rust_isRLE(one_byte.as_ptr(), 0) }, 1);
|
|
assert_eq!(unsafe { ZSTD_rust_isRLE(one_byte.as_ptr(), 1) }, 1);
|
|
|
|
let unrolled_size = size_of::<usize>() * 4;
|
|
let mut repeated = vec![0xA7u8; unrolled_size + 3];
|
|
assert_eq!(
|
|
unsafe { ZSTD_rust_isRLE(repeated.as_ptr(), repeated.len()) },
|
|
1
|
|
);
|
|
|
|
repeated[1] ^= 1;
|
|
assert_eq!(
|
|
unsafe { ZSTD_rust_isRLE(repeated.as_ptr(), repeated.len()) },
|
|
0
|
|
);
|
|
repeated[1] ^= 1;
|
|
repeated[3] ^= 1;
|
|
assert_eq!(
|
|
unsafe { ZSTD_rust_isRLE(repeated.as_ptr(), repeated.len()) },
|
|
0
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn maybe_rle_uses_the_original_sequence_and_literal_thresholds() {
|
|
let mut sequences = [SeqDef::default(); 4];
|
|
let mut literals = [0u8; 10];
|
|
let sequences_start = sequences.as_mut_ptr();
|
|
let literals_start = literals.as_mut_ptr();
|
|
let mut seq_store = SeqStore_t {
|
|
sequencesStart: sequences_start,
|
|
sequences: sequences_start,
|
|
litStart: literals_start,
|
|
lit: literals_start,
|
|
llCode: std::ptr::null_mut(),
|
|
mlCode: std::ptr::null_mut(),
|
|
ofCode: std::ptr::null_mut(),
|
|
maxNbSeq: sequences.len(),
|
|
maxNbLit: literals.len(),
|
|
longLengthType: 0,
|
|
longLengthPos: 0,
|
|
};
|
|
|
|
assert_eq!(unsafe { ZSTD_rust_maybeRLE(&seq_store) }, 1);
|
|
seq_store.sequences = unsafe { sequences_start.add(3) };
|
|
seq_store.lit = unsafe { literals_start.add(9) };
|
|
assert_eq!(unsafe { ZSTD_rust_maybeRLE(&seq_store) }, 1);
|
|
seq_store.sequences = unsafe { sequences_start.add(4) };
|
|
assert_eq!(unsafe { ZSTD_rust_maybeRLE(&seq_store) }, 0);
|
|
seq_store.sequences = unsafe { sequences_start.add(3) };
|
|
seq_store.lit = unsafe { literals_start.add(10) };
|
|
assert_eq!(unsafe { ZSTD_rust_maybeRLE(&seq_store) }, 0);
|
|
}
|
|
|
|
#[test]
|
|
fn sequence_store_byte_counts_include_the_side_band_long_length() {
|
|
let mut sequences = [
|
|
SeqDef {
|
|
offBase: 4,
|
|
litLength: 2,
|
|
mlBase: 4,
|
|
},
|
|
SeqDef {
|
|
offBase: 5,
|
|
litLength: 5,
|
|
mlBase: 5,
|
|
},
|
|
SeqDef {
|
|
offBase: 6,
|
|
litLength: 7,
|
|
mlBase: 6,
|
|
},
|
|
];
|
|
let mut literals = [0u8; 32];
|
|
let sequences_start = sequences.as_mut_ptr();
|
|
let literals_start = literals.as_mut_ptr();
|
|
let mut seq_store = SeqStore_t {
|
|
sequencesStart: sequences_start,
|
|
sequences: unsafe { sequences_start.add(sequences.len()) },
|
|
litStart: literals_start,
|
|
lit: unsafe { literals_start.add(14) },
|
|
llCode: std::ptr::null_mut(),
|
|
mlCode: std::ptr::null_mut(),
|
|
ofCode: std::ptr::null_mut(),
|
|
maxNbSeq: sequences.len(),
|
|
maxNbLit: literals.len(),
|
|
longLengthType: ZSTD_LLT_LITERAL_LENGTH,
|
|
longLengthPos: 1,
|
|
};
|
|
|
|
assert_eq!(
|
|
unsafe { ZSTD_rust_countSeqStoreLiteralsBytes(&seq_store) },
|
|
2 + 5 + 7 + 0x10000
|
|
);
|
|
assert_eq!(
|
|
unsafe { ZSTD_rust_countSeqStoreMatchBytes(&seq_store) },
|
|
(4 + MINMATCH) + (5 + MINMATCH) + (6 + MINMATCH)
|
|
);
|
|
|
|
seq_store.longLengthType = ZSTD_LLT_MATCH_LENGTH;
|
|
assert_eq!(
|
|
unsafe { ZSTD_rust_countSeqStoreLiteralsBytes(&seq_store) },
|
|
2 + 5 + 7
|
|
);
|
|
assert_eq!(
|
|
unsafe { ZSTD_rust_countSeqStoreMatchBytes(&seq_store) },
|
|
(4 + MINMATCH) + (5 + MINMATCH) + (6 + MINMATCH) + 0x10000
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn derive_seq_store_chunk_shifts_views_and_long_length_position() {
|
|
let mut sequences = [
|
|
SeqDef {
|
|
offBase: 4,
|
|
litLength: 2,
|
|
mlBase: 0,
|
|
},
|
|
SeqDef {
|
|
offBase: 5,
|
|
litLength: 5,
|
|
mlBase: 1,
|
|
},
|
|
SeqDef {
|
|
offBase: 6,
|
|
litLength: 7,
|
|
mlBase: 2,
|
|
},
|
|
SeqDef {
|
|
offBase: 7,
|
|
litLength: 3,
|
|
mlBase: 3,
|
|
},
|
|
];
|
|
let mut literals = [0u8; 20];
|
|
let mut ll_codes = [10u8; 4];
|
|
let mut ml_codes = [20u8; 4];
|
|
let mut of_codes = [30u8; 4];
|
|
let sequences_start = sequences.as_mut_ptr();
|
|
let literals_start = literals.as_mut_ptr();
|
|
let original = SeqStore_t {
|
|
sequencesStart: sequences_start,
|
|
sequences: unsafe { sequences_start.add(sequences.len()) },
|
|
litStart: literals_start,
|
|
lit: unsafe { literals_start.add(17) },
|
|
llCode: ll_codes.as_mut_ptr(),
|
|
mlCode: ml_codes.as_mut_ptr(),
|
|
ofCode: of_codes.as_mut_ptr(),
|
|
maxNbSeq: sequences.len(),
|
|
maxNbLit: literals.len(),
|
|
longLengthType: ZSTD_LLT_MATCH_LENGTH,
|
|
longLengthPos: 2,
|
|
};
|
|
let mut result = SeqStore_t {
|
|
sequencesStart: std::ptr::null_mut(),
|
|
sequences: std::ptr::null_mut(),
|
|
litStart: std::ptr::null_mut(),
|
|
lit: std::ptr::null_mut(),
|
|
llCode: std::ptr::null_mut(),
|
|
mlCode: std::ptr::null_mut(),
|
|
ofCode: std::ptr::null_mut(),
|
|
maxNbSeq: 0,
|
|
maxNbLit: 0,
|
|
longLengthType: 0,
|
|
longLengthPos: 0,
|
|
};
|
|
|
|
unsafe {
|
|
ZSTD_rust_deriveSeqStoreChunk(&mut result, &original, 1, 3);
|
|
}
|
|
assert_eq!(result.sequencesStart, unsafe { sequences_start.add(1) });
|
|
assert_eq!(result.sequences, unsafe { sequences_start.add(3) });
|
|
assert_eq!(result.litStart, unsafe { literals_start.add(2) });
|
|
assert_eq!(result.lit, unsafe { literals_start.add(14) });
|
|
assert_eq!(result.llCode, unsafe { ll_codes.as_mut_ptr().add(1) });
|
|
assert_eq!(result.mlCode, unsafe { ml_codes.as_mut_ptr().add(1) });
|
|
assert_eq!(result.ofCode, unsafe { of_codes.as_mut_ptr().add(1) });
|
|
assert_eq!(result.longLengthType, ZSTD_LLT_MATCH_LENGTH);
|
|
assert_eq!(result.longLengthPos, 1);
|
|
assert_eq!(result.maxNbSeq, original.maxNbSeq);
|
|
assert_eq!(result.maxNbLit, original.maxNbLit);
|
|
assert_eq!(
|
|
unsafe { ZSTD_rust_countSeqStoreMatchBytes(&result) },
|
|
(1 + MINMATCH) + (2 + MINMATCH) + 0x10000
|
|
);
|
|
|
|
unsafe {
|
|
ZSTD_rust_deriveSeqStoreChunk(&mut result, &original, 3, 4);
|
|
}
|
|
assert_eq!(result.sequencesStart, unsafe { sequences_start.add(3) });
|
|
assert_eq!(result.sequences, unsafe { sequences_start.add(4) });
|
|
assert_eq!(result.litStart, unsafe { literals_start.add(14) });
|
|
assert_eq!(result.lit, original.lit);
|
|
assert_eq!(result.longLengthType, 0);
|
|
assert_eq!(result.longLengthPos, original.longLengthPos);
|
|
}
|
|
|
|
#[test]
|
|
fn resolve_repcode_to_raw_offset_matches_c_repcode_numbering() {
|
|
let reps = [11u32, 22, 33];
|
|
assert_eq!(
|
|
unsafe { ZSTD_rust_resolveRepcodeToRawOffset(reps.as_ptr(), 1, 0) },
|
|
11
|
|
);
|
|
assert_eq!(
|
|
unsafe { ZSTD_rust_resolveRepcodeToRawOffset(reps.as_ptr(), 1, 1) },
|
|
22
|
|
);
|
|
assert_eq!(
|
|
unsafe { ZSTD_rust_resolveRepcodeToRawOffset(reps.as_ptr(), 2, 0) },
|
|
22
|
|
);
|
|
assert_eq!(
|
|
unsafe { ZSTD_rust_resolveRepcodeToRawOffset(reps.as_ptr(), 2, 1) },
|
|
33
|
|
);
|
|
assert_eq!(
|
|
unsafe { ZSTD_rust_resolveRepcodeToRawOffset(reps.as_ptr(), 3, 0) },
|
|
33
|
|
);
|
|
assert_eq!(
|
|
unsafe { ZSTD_rust_resolveRepcodeToRawOffset(reps.as_ptr(), 3, 1) },
|
|
10
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn repcode_updates_match_the_c_sum_type_rules() {
|
|
let mut reps = [1, 4, 8];
|
|
update_rep(&mut reps, 10, false);
|
|
assert_eq!(reps, [7, 1, 4]);
|
|
update_rep(&mut reps, 1, true);
|
|
assert_eq!(reps, [1, 7, 4]);
|
|
update_rep(&mut reps, 2, false);
|
|
assert_eq!(reps, [7, 1, 4]);
|
|
update_rep(&mut reps, 3, false);
|
|
assert_eq!(reps, [4, 7, 1]);
|
|
}
|
|
|
|
#[test]
|
|
fn seq_to_codes_matches_the_reference_tables() {
|
|
let mut store = TestSeqStore::new(
|
|
vec![
|
|
SeqDef {
|
|
offBase: 4,
|
|
litLength: 0,
|
|
mlBase: 0,
|
|
},
|
|
SeqDef {
|
|
offBase: 1,
|
|
litLength: 63,
|
|
mlBase: 127,
|
|
},
|
|
SeqDef {
|
|
offBase: 1 << 20,
|
|
litLength: 64,
|
|
mlBase: 128,
|
|
},
|
|
],
|
|
vec![0; 127],
|
|
);
|
|
let seq_store = store.seq_store();
|
|
let long_offsets = unsafe { ZSTD_seqToCodes(&seq_store) };
|
|
assert_eq!(long_offsets, 0);
|
|
assert_eq!(store.ll_codes, [0, 24, 25]);
|
|
assert_eq!(store.ml_codes, [0, 42, 43]);
|
|
assert_eq!(store.of_codes, [2, 0, 20]);
|
|
}
|
|
|
|
#[test]
|
|
fn rle_sequences_produce_one_byte_table_headers() {
|
|
// Five identical sequences select set_rle for all three symbol kinds;
|
|
// each table header is then exactly the one RLE symbol byte.
|
|
let mut store = TestSeqStore::new(
|
|
vec![
|
|
SeqDef {
|
|
offBase: 8,
|
|
litLength: 2,
|
|
mlBase: 3,
|
|
};
|
|
5
|
|
],
|
|
vec![b'x'; 10],
|
|
);
|
|
let seq_store = store.seq_store();
|
|
let prev = empty_fse_tables(FSE_REPEAT_NONE);
|
|
let mut next = empty_fse_tables(FSE_REPEAT_NONE);
|
|
let mut metadata =
|
|
unsafe { std::mem::MaybeUninit::<ZSTD_fseCTablesMetadata_t>::zeroed().assume_init() };
|
|
let mut workspace = vec![0u64; ENTROPY_WORKSPACE_SIZE / size_of::<u64>()];
|
|
|
|
let size = unsafe {
|
|
build_block_entropy_stats_sequences(
|
|
&seq_store,
|
|
&prev,
|
|
&mut next,
|
|
1,
|
|
&mut metadata,
|
|
workspace.as_mut_ptr().cast::<c_void>(),
|
|
ENTROPY_WORKSPACE_SIZE,
|
|
)
|
|
};
|
|
assert_eq!(size, 3);
|
|
assert_eq!(metadata.llType, SET_RLE);
|
|
assert_eq!(metadata.ofType, SET_RLE);
|
|
assert_eq!(metadata.mlType, SET_RLE);
|
|
assert_eq!(metadata.lastCountSize, 0);
|
|
assert_eq!(&metadata.fseTablesBuffer[..3], &[2, 3, 3]);
|
|
assert_eq!(next.litlength_repeatMode, FSE_REPEAT_NONE);
|
|
assert_eq!(next.offcode_repeatMode, FSE_REPEAT_NONE);
|
|
assert_eq!(next.matchlength_repeatMode, FSE_REPEAT_NONE);
|
|
}
|
|
|
|
#[test]
|
|
fn empty_seq_store_emits_raw_literals_and_repeats_tables() {
|
|
let mut store = TestSeqStore::new(Vec::new(), b"hello".to_vec());
|
|
let seq_store = store.seq_store();
|
|
let prev = ZSTD_entropyCTables_t {
|
|
huf: empty_huf_tables(HUF_REPEAT_NONE),
|
|
fse: empty_fse_tables(2),
|
|
};
|
|
let mut next = ZSTD_entropyCTables_t {
|
|
huf: empty_huf_tables(HUF_REPEAT_NONE),
|
|
fse: empty_fse_tables(FSE_REPEAT_NONE),
|
|
};
|
|
let mut workspace = vec![0u64; ENTROPY_WORKSPACE_SIZE / size_of::<u64>()];
|
|
let mut output = [0xA5u8; 32];
|
|
|
|
let size = unsafe {
|
|
ZSTD_rust_entropyCompressSeqStore_internal(
|
|
output.as_mut_ptr().cast::<c_void>(),
|
|
output.len(),
|
|
seq_store.litStart.cast::<c_void>(),
|
|
5,
|
|
&seq_store,
|
|
&prev,
|
|
&mut next,
|
|
1,
|
|
0,
|
|
workspace.as_mut_ptr().cast::<c_void>(),
|
|
ENTROPY_WORKSPACE_SIZE,
|
|
0,
|
|
)
|
|
};
|
|
// Raw literals header (5 << 3), five literal bytes, nbSeq == 0.
|
|
assert_eq!(size, 7);
|
|
assert_eq!(&output[..7], &[0x28, b'h', b'e', b'l', b'l', b'o', 0x00]);
|
|
// With no sequences, the previous FSE tables are copied unchanged.
|
|
assert_eq!(next.fse.offcode_repeatMode, 2);
|
|
assert_eq!(next.fse.matchlength_repeatMode, 2);
|
|
assert_eq!(next.fse.litlength_repeatMode, 2);
|
|
}
|
|
|
|
#[test]
|
|
fn literal_stats_pick_rle_and_compressed_types() {
|
|
let mut workspace = vec![0u64; ENTROPY_WORKSPACE_SIZE / size_of::<u64>()];
|
|
let prev = empty_huf_tables(HUF_REPEAT_NONE);
|
|
let mut next = empty_huf_tables(HUF_REPEAT_NONE);
|
|
let mut metadata =
|
|
unsafe { std::mem::MaybeUninit::<ZSTD_hufCTablesMetadata_t>::zeroed().assume_init() };
|
|
|
|
let rle = [7u8; 128];
|
|
let size = unsafe {
|
|
build_block_entropy_stats_literals(
|
|
rle.as_ptr().cast::<c_void>(),
|
|
rle.len(),
|
|
&prev,
|
|
&mut next,
|
|
&mut metadata,
|
|
0,
|
|
workspace.as_mut_ptr().cast::<c_void>(),
|
|
ENTROPY_WORKSPACE_SIZE,
|
|
0,
|
|
)
|
|
};
|
|
assert_eq!(size, 0);
|
|
assert_eq!(metadata.hType, SET_RLE);
|
|
|
|
let mut compressible = [0u8; 1024];
|
|
for (index, byte) in compressible.iter_mut().enumerate() {
|
|
*byte = (index % 4) as u8;
|
|
}
|
|
let size = unsafe {
|
|
build_block_entropy_stats_literals(
|
|
compressible.as_ptr().cast::<c_void>(),
|
|
compressible.len(),
|
|
&prev,
|
|
&mut next,
|
|
&mut metadata,
|
|
0,
|
|
workspace.as_mut_ptr().cast::<c_void>(),
|
|
ENTROPY_WORKSPACE_SIZE,
|
|
0,
|
|
)
|
|
};
|
|
assert!(!ERR_isError(size));
|
|
assert!(size > 0);
|
|
assert_eq!(metadata.hType, SET_COMPRESSED);
|
|
assert_eq!(metadata.hufDesSize, 0); // caller stores the returned size
|
|
assert_eq!(next.repeatMode, HUF_REPEAT_CHECK);
|
|
}
|
|
|
|
#[test]
|
|
fn copied_sequences_resolve_repcodes_and_append_the_delimiter() {
|
|
let mut store = TestSeqStore::new(
|
|
vec![
|
|
SeqDef {
|
|
offBase: 3 + 100, // raw offset 100
|
|
litLength: 4,
|
|
mlBase: 5,
|
|
},
|
|
SeqDef {
|
|
offBase: 1, // repcode 1 with literals: rep[0] == 100
|
|
litLength: 2,
|
|
mlBase: 7,
|
|
},
|
|
SeqDef {
|
|
offBase: 3, // repcode 3 without literals: rep[0] - 1 == 99
|
|
litLength: 0,
|
|
mlBase: 9,
|
|
},
|
|
],
|
|
vec![0; 10],
|
|
);
|
|
let seq_store = store.seq_store();
|
|
let mut out = [ZSTD_Sequence {
|
|
offset: 0xAA,
|
|
litLength: 0xAA,
|
|
matchLength: 0xAA,
|
|
rep: 0xAA,
|
|
}; 8];
|
|
let mut collector = SeqCollector {
|
|
collectSequences: 1,
|
|
seqStart: out.as_mut_ptr(),
|
|
seqIndex: 1,
|
|
maxSequences: 8,
|
|
};
|
|
let reps = [1u32, 4, 8];
|
|
|
|
let result =
|
|
unsafe { ZSTD_rust_copyBlockSequences(&mut collector, &seq_store, reps.as_ptr()) };
|
|
assert_eq!(result, 0);
|
|
assert_eq!(collector.seqIndex, 5);
|
|
let expect = [
|
|
(100, 4, 5 + 3, 0),
|
|
(100, 2, 7 + 3, 1),
|
|
(99, 0, 9 + 3, 3),
|
|
(0, 10 - 6, 0, 0xAA), // delimiter: last literals; rep untouched
|
|
];
|
|
for (index, &(offset, lit, matched, rep)) in expect.iter().enumerate() {
|
|
assert_eq!(out[1 + index].offset, offset, "offset {index}");
|
|
assert_eq!(out[1 + index].litLength, lit, "litLength {index}");
|
|
assert_eq!(out[1 + index].matchLength, matched, "matchLength {index}");
|
|
assert_eq!(out[1 + index].rep, rep, "rep {index}");
|
|
}
|
|
|
|
collector.maxSequences = 5;
|
|
let result =
|
|
unsafe { ZSTD_rust_copyBlockSequences(&mut collector, &seq_store, reps.as_ptr()) };
|
|
assert_eq!(result, ERROR(ZstdErrorCode::DstSizeTooSmall));
|
|
}
|
|
}
|