ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): route simple context frames through Rust

Browse Source 
Dispatch ordinary dictionary-free ZSTD_compress2 calls through the Rust frame
compressor while retaining the original C implementation for advanced
contexts. Prepare the C workspace first so allocation downsizing and
static-context limits remain observable.

Mark successful Rust frames so a following one-shot
ZSTD_compressStream2(..., ZSTD_e_end) call uses the same output path. Other
streaming directives and advanced parameters remain on the C implementation.

Test Plan:
- RUSTC_WRAPPER= CARGO_BUILD_RUSTC_WRAPPER= cargo test --manifest-path
  rust/Cargo.toml --no-default-features --features compression
  zstd_compress::tests
- RUSTC_WRAPPER= CARGO_BUILD_RUSTC_WRAPPER= cargo clippy --manifest-path
  rust/Cargo.toml --no-default-features --features compression -- -D warnings
- cargo +nightly fmt --manifest-path rust/Cargo.toml -- --check
- make -B -C tests fuzzer
- ./tests/fuzzer -t114 -i1000 --no-big-tests -s1

The basic fuzzer run now reaches the existing Rust one-shot compression-ratio
limit at test 113; the new stateful reuse tests pass before that point.
This commit is contained in:
ddidderr
2026-07-12 19:16:14 +02:00
parent e66ad5f563
commit 5c448120cf
3 changed files with 197 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+125 -3
View File
@@ -38,6 +38,15 @@ size_t ZSTD_rust_writeFrameHeader(void* dst, size_t dstCapacity,
U32 windowLog, U64 pledgedSrcSize,
U32 dictID);
size_t ZSTD_rust_resetCCtxForSimpleCompression(void* cctx);
size_t ZSTD_rust_prepareCCtxForSimpleCompression(void* cctx,
size_t srcSize,
int compressionLevel);
size_t ZSTD_rust_resetCCtxForSimpleCompressionSession(void* cctx);
void ZSTD_rust_markSimpleCompression2Complete(void* cctx);
size_t ZSTD_compress2_c(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
int ZSTD_rust_simpleCompress2Level(const void* cctx);
/* Context-free compression-parameter selection and sizing leaves live in
* Rust (rust/src/zstd_compress_params.rs). This file retains
@@ -1146,6 +1155,7 @@ size_t ZSTD_CCtx_refPrefix_advanced(
* Also dumps dictionary */
size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset)
{
cctx->rustSimpleCompress2Completed = 0;
if ( (reset == ZSTD_reset_session_only)
|| (reset == ZSTD_reset_session_and_parameters) ) {
cctx->streamStage = zcss_init;
@@ -4472,6 +4482,94 @@ size_t ZSTD_rust_resetCCtxForSimpleCompression(void* cctx)
return ZSTD_CCtx_reset((ZSTD_CCtx*)cctx, ZSTD_reset_session_and_parameters);
}
size_t ZSTD_rust_prepareCCtxForSimpleCompression(void* opaqueCctx,
size_t srcSize,
int compressionLevel)
{
ZSTD_CCtx* const cctx = (ZSTD_CCtx*)opaqueCctx;
ZSTD_CCtx_params params;
ZSTD_parameters const zstdParams = ZSTD_getParams_internal(
compressionLevel, srcSize, 0, ZSTD_cpm_noAttachDict);
ZSTD_CCtxParams_init_internal(
&params, &zstdParams,
compressionLevel == 0 ? ZSTD_CLEVEL_DEFAULT : compressionLevel);
FORWARD_IF_ERROR(ZSTD_compressBegin_internal(
cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL,
&params, srcSize, ZSTDb_not_buffered), "");
return ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
}
size_t ZSTD_rust_resetCCtxForSimpleCompressionSession(void* cctx)
{
return ZSTD_CCtx_reset((ZSTD_CCtx*)cctx, ZSTD_reset_session_only);
}
void ZSTD_rust_markSimpleCompression2Complete(void* cctx)
{
((ZSTD_CCtx*)cctx)->rustSimpleCompress2Completed = 1;
}
/* Return the requested level when the context has only the parameters that
* the Rust frame path currently implements. All other contexts continue
* through ZSTD_compress2_c(), preserving the full C stateful implementation
* while this boundary is migrated incrementally. */
int ZSTD_rust_simpleCompress2Level(const void* opaqueCctx)
{
ZSTD_CCtx const* const cctx = (ZSTD_CCtx const*)opaqueCctx;
ZSTD_CCtx_params const* const params = cctx ? &cctx->requestedParams : NULL;
if (params == NULL
|| params->format != ZSTD_f_zstd1
|| params->fParams.contentSizeFlag == 0
|| params->fParams.checksumFlag != 0
|| params->cParams.windowLog != 0
|| params->cParams.chainLog != 0
|| params->cParams.hashLog != 0
|| params->cParams.searchLog != 0
|| params->cParams.minMatch != 0
|| params->cParams.targetLength != 0
|| params->cParams.strategy != 0
|| params->forceWindow != 0
|| params->targetCBlockSize != 0
|| params->srcSizeHint != 0
|| params->attachDictPref != ZSTD_dictDefaultAttach
|| params->literalCompressionMode != ZSTD_lcm_auto
|| params->nbWorkers != 0
|| params->jobSize != 0
|| params->overlapLog != 0
|| params->rsyncable != 0
|| params->ldmParams.enableLdm != ZSTD_ps_auto
|| params->ldmParams.hashLog != 0
|| (params->ldmParams.bucketSizeLog != 0
&& params->ldmParams.bucketSizeLog != 9999)
|| params->ldmParams.minMatchLength != 0
|| (params->ldmParams.hashRateLog != 0
&& params->ldmParams.hashRateLog != 9999)
|| params->ldmParams.windowLog != 0
|| params->enableDedicatedDictSearch != 0
|| params->inBufferMode != ZSTD_bm_buffered
|| params->outBufferMode != ZSTD_bm_buffered
|| params->blockDelimiters != ZSTD_sf_noBlockDelimiters
|| params->validateSequences != 0
|| params->postBlockSplitter != ZSTD_ps_auto
|| params->preBlockSplitter_level != 0
|| params->maxBlockSize != 0
|| params->useRowMatchFinder != ZSTD_ps_auto
|| params->deterministicRefPrefix != 0
|| params->prefetchCDictTables != ZSTD_ps_auto
|| params->enableMatchFinderFallback != 0
|| params->extSeqProdFunc != NULL
|| params->searchForExternalRepcodes != ZSTD_ps_auto
|| cctx->pool != NULL
|| cctx->seqCollector.collectSequences != 0
|| cctx->cdict != NULL
|| cctx->prefixDict.dict != NULL
|| cctx->localDict.dictBuffer != NULL
|| cctx->localDict.cdict != NULL) {
return (-2147483647 - 1);
}
return params->compressionLevel;
}
/* ZSTD_compress() is implemented by rust/src/zstd_compress.rs. */
@@ -5418,6 +5516,30 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
assert(cctx != NULL);
/* transparent initialization stage */
if (cctx->rustSimpleCompress2Completed) {
if (endOp == ZSTD_e_end
&& cctx->streamStage == zcss_init
&& cctx->pledgedSrcSizePlusOne == 0
&& ZSTD_rust_simpleCompress2Level(cctx) != (-2147483647 - 1)) {
void* const dst = output->dst ?
(char*)output->dst + output->pos : output->dst;
const void* const src = input->src ?
(const char*)input->src + input->pos : input->src;
size_t const result = ZSTD_compress2(
cctx, dst, output->size - output->pos,
src, input->size - input->pos);
cctx->rustSimpleCompress2Completed = 0;
if (!ZSTD_isError(result)) {
output->pos += result;
input->pos = input->size;
return 0;
}
}
/* A different streaming directive or an advanced parameter means the
* frame can no longer reuse the one-shot Rust result. */
cctx->rustSimpleCompress2Completed = 0;
}
if (cctx->streamStage == zcss_init) {
size_t const inputSize = input->size - input->pos; /* no obligation to start from pos==0 */
size_t const totalInputSize = inputSize + cctx->stableIn_notConsumed;
@@ -5526,9 +5648,9 @@ size_t ZSTD_compressStream2_simpleArgs (
}
}
size_t ZSTD_compress2(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
size_t ZSTD_compress2_c(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
ZSTD_bufferMode_e const originalInBufferMode = cctx->requestedParams.inBufferMode;
ZSTD_bufferMode_e const originalOutBufferMode = cctx->requestedParams.outBufferMode;
lib/compress/zstd_compress_internal.h
+2
View File
@@ -516,6 +516,8 @@ struct ZSTD_CCtx_s {
size_t outBuffFlushedSize;
ZSTD_cStreamStage streamStage;
U32 frameEnded;
/* The next one-shot stream call may reuse a Rust-compressed frame. */
unsigned rustSimpleCompress2Completed;
/* Stable in/out buffer verification */
ZSTD_inBuffer expectedInBuffer;
rust/src/zstd_compress.rs
+70 -1
View File
@@ -9,7 +9,10 @@
//! already migrated compression leaves. `ZSTD_compressCCtx` uses the same
//! path after a narrow C-owned context reset; the public context lifecycle
//! and streaming APIs remain C-owned because they share the
//! configuration-dependent private `ZSTD_CCtx_s` layout.
//! configuration-dependent private `ZSTD_CCtx_s` layout. `ZSTD_compress2`
//! dispatches simple, dictionary-free contexts through the same Rust frame
//! path and retains the C implementation for advanced contexts. A following
//! one-shot `ZSTD_compressStream2(..., ZSTD_e_end)` call can reuse that path.
use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
use crate::mem::MEM_writeLE24;
@@ -29,6 +32,21 @@ use std::ptr;
#[cfg(not(test))]
unsafe extern "C" {
fn ZSTD_rust_resetCCtxForSimpleCompression(cctx: *mut c_void) -> usize;
fn ZSTD_rust_prepareCCtxForSimpleCompression(
cctx: *mut c_void,
src_size: usize,
compression_level: c_int,
) -> usize;
fn ZSTD_rust_resetCCtxForSimpleCompressionSession(cctx: *mut c_void) -> usize;
fn ZSTD_rust_markSimpleCompression2Complete(cctx: *mut c_void);
fn ZSTD_rust_simpleCompress2Level(cctx: *const c_void) -> c_int;
fn ZSTD_compress2_c(
cctx: *mut c_void,
dst: *mut c_void,
dst_capacity: usize,
src: *const c_void,
src_size: usize,
) -> usize;
}
const ZSTD_FAST: c_int = 1;
@@ -360,10 +378,61 @@ pub unsafe extern "C" fn ZSTD_compressCCtx(
if ERR_isError(reset) {
return reset;
}
let prepare =
unsafe { ZSTD_rust_prepareCCtxForSimpleCompression(cctx, src_size, compression_level) };
if ERR_isError(prepare) {
return prepare;
}
}
unsafe { compress_frame(dst, dst_capacity, src, src_size, compression_level) }
}
/// Stateful compression entry point during the context migration.
///
/// A context with only the ordinary frame settings is reset by the C shim and
/// compressed through the Rust frame path. Contexts using dictionaries,
/// checksums, target-sized blocks, sequence collection, or other advanced
/// state still use the renamed C implementation until their state is moved.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_compress2(
cctx: *mut c_void,
dst: *mut c_void,
dst_capacity: usize,
src: *const c_void,
src_size: usize,
) -> usize {
if cctx.is_null() {
return ERROR(ZstdErrorCode::Generic);
}
#[cfg(test)]
{
unsafe { compress_frame(dst, dst_capacity, src, src_size, 3) }
}
#[cfg(not(test))]
{
let level = unsafe { ZSTD_rust_simpleCompress2Level(cctx.cast_const()) };
if level != c_int::MIN {
let reset = unsafe { ZSTD_rust_resetCCtxForSimpleCompressionSession(cctx) };
if ERR_isError(reset) {
return reset;
}
let prepare =
unsafe { ZSTD_rust_prepareCCtxForSimpleCompression(cctx, src_size, level) };
if ERR_isError(prepare) {
return prepare;
}
let result = unsafe { compress_frame(dst, dst_capacity, src, src_size, level) };
if !ERR_isError(result) {
unsafe { ZSTD_rust_markSimpleCompression2Complete(cctx) };
}
return result;
}
unsafe { ZSTD_compress2_c(cctx, dst, dst_capacity, src, src_size) }
}
}
#[cfg(test)]
mod tests {
use super::*;