ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): port the public one-shot compressor to Rust

Browse Source 
Implement ZSTD_compress in Rust using the migrated parameter, match-finder,
sequence, frame, and superblock leaves. Keep the private context and streaming
entry points C-backed until their configuration-dependent context projection is
ported, and remove only the duplicate C one-shot definition.

Test Plan:
- cargo test --manifest-path rust/Cargo.toml
- cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings
- cargo +nightly fmt --manifest-path rust/Cargo.toml --all -- --check
- make -B -C tests test-rust-lib-smoke V=1
- git diff --check

Refs: rust/src/zstd_compress.rs, lib/compress/zstd_compress.c
This commit is contained in:
ddidderr
2026-07-12 09:34:08 +02:00
parent 23e8492aa7
commit f7af4cb71f
3 changed files with 391 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+1 -18
View File
@@ -4776,24 +4776,7 @@ size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
return ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel);
}
size_t ZSTD_compress(void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
int compressionLevel)
{
size_t result;
#if ZSTD_COMPRESS_HEAPMODE
ZSTD_CCtx* cctx = ZSTD_createCCtx();
RETURN_ERROR_IF(!cctx, memory_allocation, "ZSTD_createCCtx failed");
result = ZSTD_compressCCtx(cctx, dst, dstCapacity, src, srcSize, compressionLevel);
ZSTD_freeCCtx(cctx);
#else
ZSTD_CCtx ctxBody;
ZSTD_initCCtx(&ctxBody, ZSTD_defaultCMem);
result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel);
ZSTD_freeCCtxContent(&ctxBody); /* can't free ctxBody itself, as it's on stack; free only heap content */
#endif
return result;
}
/* ZSTD_compress() is implemented by rust/src/zstd_compress.rs. */
/* ===== Dictionary API ===== */
rust/src/lib.rs
+2
View File
@@ -25,6 +25,8 @@ pub mod threading;
pub mod xxhash;
pub mod zstd_common;
#[cfg(feature = "compression")]
pub mod zstd_compress;
#[cfg(feature = "compression")]
pub mod zstd_compress_api;
#[cfg(feature = "compression")]
pub mod zstd_compress_frame;
rust/src/zstd_compress.rs
+388
View File
@@ -0,0 +1,388 @@
#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(clippy::missing_safety_doc)]
#![allow(clippy::too_many_arguments)]
//! First high-level compression slice.
//!
//! The layout-independent one-shot entry point in this module drives the
//! already migrated compression leaves. The public context lifecycle,
//! `ZSTD_compressCCtx`, and streaming APIs remain C-owned: they share the
//! configuration-dependent private `ZSTD_CCtx_s` layout.
use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
use crate::zstd_compress_frame::{ZSTD_rust_writeFrameHeader, ZSTD_writeLastEmptyBlock};
use crate::zstd_compress_params::{
ZSTD_rust_params_adjustCParams, ZSTD_rust_params_maxNbSeq, ZSTD_rust_params_selectCParams,
ZSTD_RUST_CPM_NO_ATTACH_DICT, ZSTD_RUST_PS_DISABLE,
};
use crate::zstd_compress_sequences::SeqDef;
use crate::zstd_compress_stats::{SeqStore_t, ZSTD_compressedBlockState_t};
use crate::zstd_compress_superblock::ZSTD_rust_compressSuperBlock;
use std::ffi::c_void;
use std::mem::{size_of, MaybeUninit};
use std::os::raw::c_int;
use std::ptr;
const ZSTD_DFAST: c_int = 2;
const ZSTD_BLOCKSIZE_MAX: usize = 1 << 17;
const ZSTD_CONTENTSIZE_UNKNOWN: u64 = u64::MAX;
/* HUF_WORKSPACE_SIZE + (MaxSeq + 2) * sizeof(unsigned), rounded up. The
* superblock leaf also accepts the larger pre-split workspace, so a fixed
* 16 KiB buffer is sufficient for this first non-splitting path on both
* supported pointer widths. */
const TMP_WORKSPACE_SIZE: usize = 16 << 10;
#[inline]
fn zeroed_state() -> ZSTD_compressedBlockState_t {
/* The state contains only integer arrays and enum fields. */
unsafe { MaybeUninit::<ZSTD_compressedBlockState_t>::zeroed().assume_init() }
}
#[inline]
fn checked_table_size(log: u32) -> Option<usize> {
1usize.checked_shl(log)
}
#[inline]
fn ceil_log2(size: usize) -> u32 {
if size <= 1 {
0
} else {
usize::BITS - (size - 1).leading_zeros()
}
}
/// Compress one frame using the already migrated block leaves.
///
/// This path deliberately starts a fresh match table for each 128 KiB block.
/// That keeps the Rust-owned context independent from the still-private C
/// `ZSTD_MatchState_t` window while producing ordinary zstd blocks that any
/// decoder can consume.
unsafe fn compress_frame(
dst: *mut c_void,
dst_capacity: usize,
src: *const c_void,
src_size: usize,
compression_level: c_int,
) -> usize {
if dst.is_null() {
return ERROR(if dst_capacity == 0 {
ZstdErrorCode::DstSizeTooSmall
} else {
ZstdErrorCode::DstBufferNull
});
}
if src_size != 0 && src.is_null() {
return ERROR(ZstdErrorCode::SrcSizeWrong);
}
if src_size as u64 == ZSTD_CONTENTSIZE_UNKNOWN {
return ERROR(ZstdErrorCode::SrcSizeWrong);
}
let mut cparams = ZSTD_rust_params_selectCParams(
compression_level,
src_size as u64,
0,
ZSTD_RUST_CPM_NO_ATTACH_DICT,
);
cparams = ZSTD_rust_params_adjustCParams(
cparams,
src_size as u64,
0,
ZSTD_RUST_CPM_NO_ATTACH_DICT,
ZSTD_RUST_PS_DISABLE,
);
let header_size = unsafe {
ZSTD_rust_writeFrameHeader(
dst,
dst_capacity,
0, /* noDictIDFlag */
0, /* checksumFlag */
1, /* contentSizeFlag */
0, /* zstd frame */
cparams.windowLog,
src_size as u64,
0,
)
};
if ERR_isError(header_size) {
return header_size;
}
if src_size == 0 {
let empty_block = unsafe {
ZSTD_writeLastEmptyBlock(
dst.cast::<u8>().add(header_size).cast(),
dst_capacity - header_size,
)
};
return if ERR_isError(empty_block) {
empty_block
} else {
header_size + empty_block
};
}
let first_block_size = src_size.min(ZSTD_BLOCKSIZE_MAX);
/* A table larger than the block cannot contain a useful index for this
* independent-block path. This also keeps one-shot small inputs from
* allocating the table selected for a huge source-size hint. */
let matcher_hash_log = cparams.hashLog.min(ceil_log2(first_block_size).max(6));
let matcher_chain_log = cparams.chainLog.min(ceil_log2(first_block_size).max(6));
let hash_size = match checked_table_size(matcher_hash_log) {
Some(size) => size,
None => return ERROR(ZstdErrorCode::MemoryAllocation),
};
let chain_size = match checked_table_size(matcher_chain_log) {
Some(size) => size,
None => return ERROR(ZstdErrorCode::MemoryAllocation),
};
let max_nb_seq =
ZSTD_rust_params_maxNbSeq(ZSTD_BLOCKSIZE_MAX, cparams.minMatch, 0).saturating_add(1);
let mut sequences = vec![SeqDef::default(); max_nb_seq];
let mut literals = vec![0u8; ZSTD_BLOCKSIZE_MAX];
let mut ll_codes = vec![0u8; max_nb_seq];
let mut ml_codes = vec![0u8; max_nb_seq];
let mut of_codes = vec![0u8; max_nb_seq];
let mut hash_table = vec![0u32; hash_size];
let mut chain_table = vec![0u32; chain_size];
let mut workspace = vec![0u64; TMP_WORKSPACE_SIZE / size_of::<u64>()];
let mut prev_block = zeroed_state();
let mut next_block = zeroed_state();
prev_block.rep = [1, 4, 8];
next_block.rep = prev_block.rep;
let seq_store = &mut SeqStore_t {
sequencesStart: sequences.as_mut_ptr(),
sequences: sequences.as_mut_ptr(),
litStart: literals.as_mut_ptr(),
lit: literals.as_mut_ptr(),
llCode: ll_codes.as_mut_ptr(),
mlCode: ml_codes.as_mut_ptr(),
ofCode: of_codes.as_mut_ptr(),
maxNbSeq: max_nb_seq,
maxNbLit: ZSTD_BLOCKSIZE_MAX,
longLengthType: 0,
longLengthPos: 0,
};
let source = src.cast::<u8>();
let output = dst.cast::<u8>();
let mut input_offset = 0usize;
let mut output_offset = header_size;
while input_offset < src_size {
let block_size = (src_size - input_offset).min(ZSTD_BLOCKSIZE_MAX);
let block_src = unsafe { source.add(input_offset) };
let block_end = unsafe { block_src.add(block_size) };
seq_store.sequences = seq_store.sequencesStart;
seq_store.lit = seq_store.litStart;
seq_store.longLengthType = 0;
seq_store.longLengthPos = 0;
next_block.rep = prev_block.rep;
let mut reps = prev_block.rep;
let last_literals = if block_size < 8 {
block_size
} else if cparams.strategy == ZSTD_DFAST {
unsafe {
crate::zstd_double_fast::ZSTD_rust_compressBlock_doubleFast(
hash_table.as_mut_ptr(),
chain_table.as_mut_ptr(),
block_src,
0,
0,
matcher_hash_log,
matcher_chain_log,
cparams.minMatch,
cparams.windowLog,
(seq_store as *mut SeqStore_t).cast(),
reps.as_mut_ptr(),
block_src.cast(),
block_size,
)
}
} else {
unsafe {
crate::zstd_fast::ZSTD_rust_compressBlock_fast(
hash_table.as_mut_ptr(),
block_src,
0,
0,
matcher_hash_log,
cparams.minMatch,
cparams.targetLength,
cparams.windowLog,
(seq_store as *mut SeqStore_t).cast(),
reps.as_mut_ptr(),
block_src.cast(),
block_size,
)
}
};
if last_literals > block_size {
return ERROR(ZstdErrorCode::Generic);
}
let last_literal_src = unsafe { block_end.sub(last_literals) };
if last_literals != 0 {
unsafe {
ptr::copy_nonoverlapping(last_literal_src, seq_store.lit, last_literals);
seq_store.lit = seq_store.lit.add(last_literals);
}
}
next_block.rep = reps;
let last_block = u32::from(input_offset + block_size == src_size);
let remaining_capacity = dst_capacity.saturating_sub(output_offset);
let written = unsafe {
ZSTD_rust_compressSuperBlock(
(seq_store as *mut SeqStore_t).cast(),
(&prev_block as *const ZSTD_compressedBlockState_t).cast(),
(&mut next_block as *mut ZSTD_compressedBlockState_t).cast(),
cparams.strategy,
0, /* literals compression enabled */
workspace.as_mut_ptr().cast(),
TMP_WORKSPACE_SIZE,
0, /* BMI2 is optional; portable Rust leaf path */
cparams.windowLog,
0, /* no target compressed block size */
output.add(output_offset).cast(),
remaining_capacity,
block_src.cast(),
block_size,
last_block,
)
};
if ERR_isError(written) {
return written;
}
if written > remaining_capacity {
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
output_offset += written;
input_offset += block_size;
std::mem::swap(&mut prev_block, &mut next_block);
hash_table.fill(0);
chain_table.fill(0);
}
output_offset
}
/// Simple one-shot compression entry point.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_compress(
dst: *mut c_void,
dst_capacity: usize,
src: *const c_void,
src_size: usize,
compression_level: c_int,
) -> usize {
unsafe { compress_frame(dst, dst_capacity, src, src_size, compression_level) }
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
use std::process::{Command, Stdio};
fn system_round_trip(compressed: &[u8]) -> Option<Vec<u8>> {
let mut child = Command::new("zstd")
.args(["-q", "-d", "-c"])
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.ok()?;
child.stdin.take()?.write_all(compressed).ok()?;
let output = child.wait_with_output().ok()?;
if !output.status.success() {
panic!(
"system zstd rejected Rust output: {}",
String::from_utf8_lossy(&output.stderr)
);
}
Some(output.stdout)
}
fn compress_input(input: &[u8], level: c_int) -> Vec<u8> {
let capacity = crate::zstd_compress_api::ZSTD_compressBound(input.len());
assert!(!ERR_isError(capacity));
let mut output = vec![0u8; capacity];
let written = unsafe {
ZSTD_compress(
output.as_mut_ptr().cast(),
output.len(),
input.as_ptr().cast(),
input.len(),
level,
)
};
assert!(!ERR_isError(written));
output.truncate(written);
output
}
#[test]
fn public_one_shot_abi_is_c_compatible() {
let entry: unsafe extern "C" fn(*mut c_void, usize, *const c_void, usize, c_int) -> usize =
ZSTD_compress;
assert_eq!(size_of::<usize>(), size_of::<*const c_void>());
let _ = entry;
}
#[test]
fn one_shot_round_trip_across_block_boundaries() {
let mut input = Vec::with_capacity(128 * 1024 + 37);
for index in 0..(128 * 1024 + 37) {
input.push(((index * 17) ^ (index / 31)) as u8);
}
let compressed = compress_input(&input, 3);
if let Some(restored) = system_round_trip(&compressed) {
assert_eq!(restored, input);
}
}
#[test]
fn empty_and_short_inputs_have_valid_frames() {
for input in [b"".as_slice(), b"a", b"abcdefg", b"abcdefgh"] {
let compressed = compress_input(input, 1);
if let Some(restored) = system_round_trip(&compressed) {
assert_eq!(restored, input);
}
}
}
#[test]
fn public_error_paths_match_size_t_error_contract() {
let mut output = [0u8; 64];
let source = [1u8; 8];
assert_eq!(
unsafe {
ZSTD_compress(
output.as_mut_ptr().cast(),
0,
source.as_ptr().cast(),
source.len(),
3,
)
},
ERROR(ZstdErrorCode::DstSizeTooSmall)
);
assert_eq!(
unsafe { ZSTD_compress(ptr::null_mut(), 1, source.as_ptr().cast(), source.len(), 3) },
ERROR(ZstdErrorCode::DstBufferNull)
);
assert_eq!(
unsafe { ZSTD_compress(output.as_mut_ptr().cast(), output.len(), ptr::null(), 1, 3) },
ERROR(ZstdErrorCode::SrcSizeWrong)
);
}
}