feat(compress): port simple context compression to Rust
Move ZSTD_compressCCtx() onto the Rust frame-compression path while retaining only the C-owned reset needed for its private context layout. The simple API ignores advanced context parameters by contract, so the Rust entry point can share the one-shot implementation after reset. Preserve valid frames when the Rust superblock leaf declines a compressed block by emitting a raw block instead. Match the fast-strategy literal policy for negative levels so compatibility behavior remains intact. Streaming, stateful ZSTD_compress2(), and multithreaded context orchestration remain C-owned for a later slice. The full fuzzer currently reaches an existing compress2() superblock-expansion failure after these simple-API checks; that stateful path is intentionally out of scope here. Test Plan: - cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression zstd_compress::tests - cargo clippy --manifest-path rust/Cargo.toml -- -D warnings (library, benches, and tests) - cargo +nightly fmt --manifest-path rust/Cargo.toml -- --check - make -B -C programs zstd V=1 - make -B -C examples multiple_simple_compression - ./multiple_simple_compression README.md ../rust/README.md
This commit is contained in:
@@ -37,6 +37,7 @@ size_t ZSTD_rust_writeFrameHeader(void* dst, size_t dstCapacity,
|
|||||||
int contentSizeFlag, int format,
|
int contentSizeFlag, int format,
|
||||||
U32 windowLog, U64 pledgedSrcSize,
|
U32 windowLog, U64 pledgedSrcSize,
|
||||||
U32 dictID);
|
U32 dictID);
|
||||||
|
size_t ZSTD_rust_resetCCtxForSimpleCompression(void* cctx);
|
||||||
|
|
||||||
/* Context-free compression-parameter selection and sizing leaves live in
|
/* Context-free compression-parameter selection and sizing leaves live in
|
||||||
* Rust (rust/src/zstd_compress_params.rs). This file retains
|
* Rust (rust/src/zstd_compress_params.rs). This file retains
|
||||||
@@ -4462,14 +4463,13 @@ size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx,
|
|||||||
return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctx->simpleApiParams);
|
return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctx->simpleApiParams);
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
|
/* ZSTD_compressCCtx() is implemented by rust/src/zstd_compress.rs. */
|
||||||
void* dst, size_t dstCapacity,
|
|
||||||
const void* src, size_t srcSize,
|
/* The Rust simple API still needs the C-owned context reset, but does not
|
||||||
int compressionLevel)
|
* cross the private context layout. */
|
||||||
|
size_t ZSTD_rust_resetCCtxForSimpleCompression(void* cctx)
|
||||||
{
|
{
|
||||||
DEBUGLOG(4, "ZSTD_compressCCtx (srcSize=%u)", (unsigned)srcSize);
|
return ZSTD_CCtx_reset((ZSTD_CCtx*)cctx, ZSTD_reset_session_and_parameters);
|
||||||
assert(cctx != NULL);
|
|
||||||
return ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ZSTD_compress() is implemented by rust/src/zstd_compress.rs. */
|
/* ZSTD_compress() is implemented by rust/src/zstd_compress.rs. */
|
||||||
|
|||||||
+115
-4
@@ -6,11 +6,13 @@
|
|||||||
//! First high-level compression slice.
|
//! First high-level compression slice.
|
||||||
//!
|
//!
|
||||||
//! The layout-independent one-shot entry point in this module drives the
|
//! The layout-independent one-shot entry point in this module drives the
|
||||||
//! already migrated compression leaves. The public context lifecycle,
|
//! already migrated compression leaves. `ZSTD_compressCCtx` uses the same
|
||||||
//! `ZSTD_compressCCtx`, and streaming APIs remain C-owned: they share the
|
//! path after a narrow C-owned context reset; the public context lifecycle
|
||||||
|
//! and streaming APIs remain C-owned because they share the
|
||||||
//! configuration-dependent private `ZSTD_CCtx_s` layout.
|
//! configuration-dependent private `ZSTD_CCtx_s` layout.
|
||||||
|
|
||||||
use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
|
use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
|
||||||
|
use crate::mem::MEM_writeLE24;
|
||||||
use crate::zstd_compress_frame::{ZSTD_rust_writeFrameHeader, ZSTD_writeLastEmptyBlock};
|
use crate::zstd_compress_frame::{ZSTD_rust_writeFrameHeader, ZSTD_writeLastEmptyBlock};
|
||||||
use crate::zstd_compress_params::{
|
use crate::zstd_compress_params::{
|
||||||
ZSTD_rust_params_adjustCParams, ZSTD_rust_params_maxNbSeq, ZSTD_rust_params_selectCParams,
|
ZSTD_rust_params_adjustCParams, ZSTD_rust_params_maxNbSeq, ZSTD_rust_params_selectCParams,
|
||||||
@@ -24,6 +26,12 @@ use std::mem::{size_of, MaybeUninit};
|
|||||||
use std::os::raw::c_int;
|
use std::os::raw::c_int;
|
||||||
use std::ptr;
|
use std::ptr;
|
||||||
|
|
||||||
|
#[cfg(not(test))]
|
||||||
|
unsafe extern "C" {
|
||||||
|
fn ZSTD_rust_resetCCtxForSimpleCompression(cctx: *mut c_void) -> usize;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ZSTD_FAST: c_int = 1;
|
||||||
const ZSTD_DFAST: c_int = 2;
|
const ZSTD_DFAST: c_int = 2;
|
||||||
const ZSTD_BLOCKSIZE_MAX: usize = 1 << 17;
|
const ZSTD_BLOCKSIZE_MAX: usize = 1 << 17;
|
||||||
const ZSTD_CONTENTSIZE_UNKNOWN: u64 = u64::MAX;
|
const ZSTD_CONTENTSIZE_UNKNOWN: u64 = u64::MAX;
|
||||||
@@ -54,6 +62,32 @@ fn ceil_log2(size: usize) -> u32 {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
unsafe fn write_raw_block(
|
||||||
|
dst: *mut u8,
|
||||||
|
dst_capacity: usize,
|
||||||
|
src: *const u8,
|
||||||
|
src_size: usize,
|
||||||
|
last_block: u32,
|
||||||
|
) -> usize {
|
||||||
|
let needed = match src_size.checked_add(3) {
|
||||||
|
Some(value) => value,
|
||||||
|
None => return ERROR(ZstdErrorCode::DstSizeTooSmall),
|
||||||
|
};
|
||||||
|
if needed > dst_capacity {
|
||||||
|
return ERROR(ZstdErrorCode::DstSizeTooSmall);
|
||||||
|
}
|
||||||
|
unsafe {
|
||||||
|
MEM_writeLE24(
|
||||||
|
dst.cast(),
|
||||||
|
last_block.wrapping_add((src_size as u32).wrapping_shl(3)),
|
||||||
|
);
|
||||||
|
if src_size != 0 {
|
||||||
|
ptr::copy_nonoverlapping(src, dst.add(3), src_size);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
needed
|
||||||
|
}
|
||||||
|
|
||||||
/// Compress one frame using the already migrated block leaves.
|
/// Compress one frame using the already migrated block leaves.
|
||||||
///
|
///
|
||||||
/// This path deliberately starts a fresh match table for each 128 KiB block.
|
/// This path deliberately starts a fresh match table for each 128 KiB block.
|
||||||
@@ -143,6 +177,8 @@ unsafe fn compress_frame(
|
|||||||
|
|
||||||
let max_nb_seq =
|
let max_nb_seq =
|
||||||
ZSTD_rust_params_maxNbSeq(ZSTD_BLOCKSIZE_MAX, cparams.minMatch, 0).saturating_add(1);
|
ZSTD_rust_params_maxNbSeq(ZSTD_BLOCKSIZE_MAX, cparams.minMatch, 0).saturating_add(1);
|
||||||
|
let disable_literal_compression =
|
||||||
|
c_int::from(cparams.strategy == ZSTD_FAST && cparams.targetLength > 0);
|
||||||
let mut sequences = vec![SeqDef::default(); max_nb_seq];
|
let mut sequences = vec![SeqDef::default(); max_nb_seq];
|
||||||
let mut literals = vec![0u8; ZSTD_BLOCKSIZE_MAX];
|
let mut literals = vec![0u8; ZSTD_BLOCKSIZE_MAX];
|
||||||
let mut ll_codes = vec![0u8; max_nb_seq];
|
let mut ll_codes = vec![0u8; max_nb_seq];
|
||||||
@@ -239,13 +275,13 @@ unsafe fn compress_frame(
|
|||||||
|
|
||||||
let last_block = u32::from(input_offset + block_size == src_size);
|
let last_block = u32::from(input_offset + block_size == src_size);
|
||||||
let remaining_capacity = dst_capacity.saturating_sub(output_offset);
|
let remaining_capacity = dst_capacity.saturating_sub(output_offset);
|
||||||
let written = unsafe {
|
let mut written = unsafe {
|
||||||
ZSTD_rust_compressSuperBlock(
|
ZSTD_rust_compressSuperBlock(
|
||||||
(seq_store as *mut SeqStore_t).cast(),
|
(seq_store as *mut SeqStore_t).cast(),
|
||||||
(&prev_block as *const ZSTD_compressedBlockState_t).cast(),
|
(&prev_block as *const ZSTD_compressedBlockState_t).cast(),
|
||||||
(&mut next_block as *mut ZSTD_compressedBlockState_t).cast(),
|
(&mut next_block as *mut ZSTD_compressedBlockState_t).cast(),
|
||||||
cparams.strategy,
|
cparams.strategy,
|
||||||
0, /* literals compression enabled */
|
disable_literal_compression,
|
||||||
workspace.as_mut_ptr().cast(),
|
workspace.as_mut_ptr().cast(),
|
||||||
TMP_WORKSPACE_SIZE,
|
TMP_WORKSPACE_SIZE,
|
||||||
0, /* BMI2 is optional; portable Rust leaf path */
|
0, /* BMI2 is optional; portable Rust leaf path */
|
||||||
@@ -261,6 +297,20 @@ unsafe fn compress_frame(
|
|||||||
if ERR_isError(written) {
|
if ERR_isError(written) {
|
||||||
return written;
|
return written;
|
||||||
}
|
}
|
||||||
|
if written == 0 {
|
||||||
|
written = unsafe {
|
||||||
|
write_raw_block(
|
||||||
|
output.add(output_offset),
|
||||||
|
remaining_capacity,
|
||||||
|
block_src,
|
||||||
|
block_size,
|
||||||
|
last_block,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
if ERR_isError(written) {
|
||||||
|
return written;
|
||||||
|
}
|
||||||
|
}
|
||||||
if written > remaining_capacity {
|
if written > remaining_capacity {
|
||||||
return ERROR(ZstdErrorCode::DstSizeTooSmall);
|
return ERROR(ZstdErrorCode::DstSizeTooSmall);
|
||||||
}
|
}
|
||||||
@@ -286,6 +336,34 @@ pub unsafe extern "C" fn ZSTD_compress(
|
|||||||
unsafe { compress_frame(dst, dst_capacity, src, src_size, compression_level) }
|
unsafe { compress_frame(dst, dst_capacity, src, src_size, compression_level) }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Simple explicit-context compression entry point.
|
||||||
|
///
|
||||||
|
/// The public contract deliberately ignores all advanced context parameters.
|
||||||
|
/// C performs the context reset because the private `ZSTD_CCtx_s` layout is
|
||||||
|
/// still configuration-dependent; the frame compressor itself is entirely
|
||||||
|
/// Rust-owned and does not inspect the context.
|
||||||
|
#[no_mangle]
|
||||||
|
pub unsafe extern "C" fn ZSTD_compressCCtx(
|
||||||
|
cctx: *mut c_void,
|
||||||
|
dst: *mut c_void,
|
||||||
|
dst_capacity: usize,
|
||||||
|
src: *const c_void,
|
||||||
|
src_size: usize,
|
||||||
|
compression_level: c_int,
|
||||||
|
) -> usize {
|
||||||
|
if cctx.is_null() {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
}
|
||||||
|
#[cfg(not(test))]
|
||||||
|
{
|
||||||
|
let reset = unsafe { ZSTD_rust_resetCCtxForSimpleCompression(cctx) };
|
||||||
|
if ERR_isError(reset) {
|
||||||
|
return reset;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
unsafe { compress_frame(dst, dst_capacity, src, src_size, compression_level) }
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
@@ -333,8 +411,41 @@ mod tests {
|
|||||||
fn public_one_shot_abi_is_c_compatible() {
|
fn public_one_shot_abi_is_c_compatible() {
|
||||||
let entry: unsafe extern "C" fn(*mut c_void, usize, *const c_void, usize, c_int) -> usize =
|
let entry: unsafe extern "C" fn(*mut c_void, usize, *const c_void, usize, c_int) -> usize =
|
||||||
ZSTD_compress;
|
ZSTD_compress;
|
||||||
|
let context_entry: unsafe extern "C" fn(
|
||||||
|
*mut c_void,
|
||||||
|
*mut c_void,
|
||||||
|
usize,
|
||||||
|
*const c_void,
|
||||||
|
usize,
|
||||||
|
c_int,
|
||||||
|
) -> usize = ZSTD_compressCCtx;
|
||||||
assert_eq!(size_of::<usize>(), size_of::<*const c_void>());
|
assert_eq!(size_of::<usize>(), size_of::<*const c_void>());
|
||||||
let _ = entry;
|
let _ = entry;
|
||||||
|
let _ = context_entry;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn explicit_context_simple_api_matches_one_shot_path() {
|
||||||
|
let input = b"explicit context compression remains a simple API";
|
||||||
|
let capacity = crate::zstd_compress_api::ZSTD_compressBound(input.len());
|
||||||
|
let mut output = vec![0u8; capacity];
|
||||||
|
let written = unsafe {
|
||||||
|
ZSTD_compressCCtx(
|
||||||
|
std::ptr::dangling_mut::<c_void>(),
|
||||||
|
output.as_mut_ptr().cast(),
|
||||||
|
output.len(),
|
||||||
|
input.as_ptr().cast(),
|
||||||
|
input.len(),
|
||||||
|
3,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
assert!(!ERR_isError(written));
|
||||||
|
output.truncate(written);
|
||||||
|
let one_shot = compress_input(input, 3);
|
||||||
|
assert_eq!(output, one_shot);
|
||||||
|
if let Some(restored) = system_round_trip(&output) {
|
||||||
|
assert_eq!(restored, input);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
Reference in New Issue
Block a user