ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(compress): move MT parameter update policy to Rust

Browse Source 
Move the next-job parameter derivation in
ZSTDMT_updateCParams_whileCompressing() behind a scalar Rust projection. Rust
now reproduces the existing CCtx-parameter derivation and restores the active
frame's saved windowLog; C retains private MT/CCtx layouts, logging, and final
writes.

Expose the preprocessor-derived block-compressor exclusion mask through the
private C boundary so reduced builds keep their authoritative configuration.
Add C/Rust layout assertions and focused tests for window preservation and
strategy-exclusion routing.

Test Plan:
- `ulimit -v 41943040; cargo +nightly fmt --manifest-path rust/Cargo.toml --all -- --check`
- `ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings`
- `ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings`
- `ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/cli/Cargo.toml --all-targets` (184 tests)
- `ulimit -v 41943040; make -j1`
- `ulimit -v 41943040; make -j1 -C tests test` (all upstream tests completed successfully)
- Standalone Rust unit linking remains unavailable without the C-owned bridge symbols; integrated build and upstream MT/API coverage passed.
This commit is contained in:
ddidderr
2026-07-20 09:07:40 +02:00
parent 870d6a2e40
commit 6d3da898de
3 changed files with 258 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+2 -2
View File
@@ -3945,7 +3945,7 @@ U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
return ZSTD_rust_params_cycleLog(hashLog, (int)strat);
}
static U32 ZSTD_getCParamsExclusionMask(void);
U32 ZSTD_getCParamsExclusionMask(void);
/** ZSTD_adjustCParams_internal() :
* optimize `cPar` for a specified input (`srcSize` and `dictSize`).
@@ -4001,7 +4001,7 @@ enum {
ZSTD_RUST_PARAMS_EXCLUDE_DFAST = 1u << 6
};
static U32 ZSTD_getCParamsExclusionMask(void)
U32 ZSTD_getCParamsExclusionMask(void)
{
U32 mask = 0;
#ifdef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR
lib/compress/zstdmt_compress.c
+62 -3
View File
@@ -25,6 +25,53 @@
#include "zstd_ldm.h"
#include "zstdmt_compress.h"
/* The Rust parameter leaf receives only this scalar snapshot. Keep the
* private ZSTDMT_CCtx and ZSTD_CCtx_params layouts in this translation unit. */
typedef struct {
int compressionLevel;
int cctxSrcSizeHint;
U64 srcSizeHint;
size_t dictSize;
int mode;
int enableLdm;
U32 ldmDefaultWindowLog;
ZSTD_compressionParameters overrides;
int useRowMatchFinder;
U32 exclusionMask;
U32 savedWindowLog;
} ZSTDMT_RustCParamsUpdateProjection;
typedef char ZSTDMT_rust_cparams_update_projection_layout[
(offsetof(ZSTDMT_RustCParamsUpdateProjection, compressionLevel) == 0
&& offsetof(ZSTDMT_RustCParamsUpdateProjection, cctxSrcSizeHint)
== sizeof(int)
&& offsetof(ZSTDMT_RustCParamsUpdateProjection, srcSizeHint)
== 2 * sizeof(int)
&& offsetof(ZSTDMT_RustCParamsUpdateProjection, dictSize) == 16
&& offsetof(ZSTDMT_RustCParamsUpdateProjection, mode)
== (sizeof(void*) == 8 ? 24 : 20)
&& offsetof(ZSTDMT_RustCParamsUpdateProjection, enableLdm)
== (sizeof(void*) == 8 ? 28 : 24)
&& offsetof(ZSTDMT_RustCParamsUpdateProjection, ldmDefaultWindowLog)
== (sizeof(void*) == 8 ? 32 : 28)
&& offsetof(ZSTDMT_RustCParamsUpdateProjection, overrides)
== (sizeof(void*) == 8 ? 36 : 32)
&& offsetof(ZSTDMT_RustCParamsUpdateProjection, useRowMatchFinder)
== (sizeof(void*) == 8 ? 64 : 60)
&& offsetof(ZSTDMT_RustCParamsUpdateProjection, exclusionMask)
== (sizeof(void*) == 8 ? 68 : 64)
&& offsetof(ZSTDMT_RustCParamsUpdateProjection, savedWindowLog)
== (sizeof(void*) == 8 ? 72 : 68)
&& sizeof(ZSTDMT_RustCParamsUpdateProjection)
== (sizeof(void*) == 8 ? 80 : 72))
? 1 : -1];
ZSTD_compressionParameters ZSTD_rust_params_updateCParamsWhileCompressing(
ZSTDMT_RustCParamsUpdateProjection projection);
/* Defined in zstd_compress.c so all C translation units use the same
* preprocessor-derived block-compressor exclusion mask. */
U32 ZSTD_getCParamsExclusionMask(void);
/* Guards code to support resizing the SeqPool.
* We will want to resize the SeqPool to save memory in the future.
* Until then, comment the code out since it is unused.
@@ -2260,9 +2307,21 @@ void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_p
DEBUGLOG(5, "ZSTDMT_updateCParams_whileCompressing (level:%i)",
compressionLevel);
mtctx->params.compressionLevel = compressionLevel;
{ ZSTD_compressionParameters cParams = ZSTD_getCParamsFromCCtxParams(cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
cParams.windowLog = saved_wlog;
mtctx->params.cParams = cParams;
{ ZSTDMT_RustCParamsUpdateProjection const projection = {
cctxParams->compressionLevel,
cctxParams->srcSizeHint,
ZSTD_CONTENTSIZE_UNKNOWN,
0,
ZSTD_cpm_noAttachDict,
cctxParams->ldmParams.enableLdm,
ZSTD_LDM_DEFAULT_WINDOW_LOG,
cctxParams->cParams,
cctxParams->useRowMatchFinder,
ZSTD_getCParamsExclusionMask(),
saved_wlog
};
mtctx->params.cParams =
ZSTD_rust_params_updateCParamsWhileCompressing(projection);
}
}
rust/src/zstd_compress_params.rs
+194 -1
View File
@@ -15,7 +15,7 @@
//! state across the ABI.
use crate::errors::{ZstdErrorCode, ERROR};
use std::mem::size_of;
use std::mem::{offset_of, size_of};
use std::os::raw::c_int;
pub const ZSTD_CONTENTSIZE_UNKNOWN: u64 = u64::MAX;
@@ -130,6 +130,98 @@ pub struct ZSTD_bounds {
pub upperBound: c_int,
}
/// Scalar snapshot used when MT compression updates parameters between jobs.
///
/// The private `ZSTD_CCtx_params` and `ZSTDMT_CCtx` layouts stay in C. This
/// projection carries the same scalar inputs as
/// `ZSTD_getCParamsFromCCtxParams()` and the active frame's window log.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTD_rust_params_updateCParamsWhileCompressingState {
pub compressionLevel: c_int,
pub cctxSrcSizeHint: c_int,
pub srcSizeHint: u64,
pub dictSize: usize,
pub mode: c_int,
pub enableLdm: c_int,
pub ldmDefaultWindowLog: u32,
pub overrides: ZSTD_compressionParameters,
pub useRowMatchFinder: c_int,
pub exclusionMask: u32,
pub savedWindowLog: u32,
}
const _: () = {
assert!(
offset_of!(
ZSTD_rust_params_updateCParamsWhileCompressingState,
compressionLevel
) == 0
);
assert!(
offset_of!(
ZSTD_rust_params_updateCParamsWhileCompressingState,
cctxSrcSizeHint
) == size_of::<c_int>()
);
assert!(
offset_of!(
ZSTD_rust_params_updateCParamsWhileCompressingState,
srcSizeHint
) == 2 * size_of::<c_int>()
);
assert!(
offset_of!(
ZSTD_rust_params_updateCParamsWhileCompressingState,
dictSize
) == 16
);
assert!(
offset_of!(ZSTD_rust_params_updateCParamsWhileCompressingState, mode)
== if size_of::<usize>() == 8 { 24 } else { 20 }
);
assert!(
offset_of!(
ZSTD_rust_params_updateCParamsWhileCompressingState,
enableLdm
) == if size_of::<usize>() == 8 { 28 } else { 24 }
);
assert!(
offset_of!(
ZSTD_rust_params_updateCParamsWhileCompressingState,
ldmDefaultWindowLog
) == if size_of::<usize>() == 8 { 32 } else { 28 }
);
assert!(
offset_of!(
ZSTD_rust_params_updateCParamsWhileCompressingState,
overrides
) == if size_of::<usize>() == 8 { 36 } else { 32 }
);
assert!(
offset_of!(
ZSTD_rust_params_updateCParamsWhileCompressingState,
useRowMatchFinder
) == if size_of::<usize>() == 8 { 64 } else { 60 }
);
assert!(
offset_of!(
ZSTD_rust_params_updateCParamsWhileCompressingState,
exclusionMask
) == if size_of::<usize>() == 8 { 68 } else { 64 }
);
assert!(
offset_of!(
ZSTD_rust_params_updateCParamsWhileCompressingState,
savedWindowLog
) == if size_of::<usize>() == 8 { 72 } else { 68 }
);
assert!(
size_of::<ZSTD_rust_params_updateCParamsWhileCompressingState>()
== if size_of::<usize>() == 8 { 80 } else { 72 }
);
};
/* `clevels.h`, represented as raw fields so the table stays compact and easy
* to compare mechanically against its C source. Field order is W, C, H, S,
* L, TL, strategy. */
@@ -969,6 +1061,25 @@ fn get_cparams_from_cctx_params(
)
}
fn update_cparams_while_compressing(
projection: ZSTD_rust_params_updateCParamsWhileCompressingState,
) -> ZSTD_compressionParameters {
let mut cparams = get_cparams_from_cctx_params(
projection.compressionLevel,
projection.cctxSrcSizeHint,
projection.srcSizeHint,
projection.dictSize,
projection.mode,
projection.enableLdm,
projection.ldmDefaultWindowLog,
projection.overrides,
projection.useRowMatchFinder,
projection.exclusionMask,
);
cparams.windowLog = projection.savedWindowLog;
cparams
}
#[inline]
fn make_params(cparams: ZSTD_compressionParameters) -> ZSTD_parameters {
ZSTD_parameters {
@@ -1195,6 +1306,16 @@ pub extern "C" fn ZSTD_rust_params_getCParamsFromCCtxParams(
)
}
/// Derives the next MT job's parameters while preserving the active frame's
/// window log. The C shim owns the private-context projection and final
/// writes; Rust owns the parameter derivation and update policy.
#[no_mangle]
pub extern "C" fn ZSTD_rust_params_updateCParamsWhileCompressing(
projection: ZSTD_rust_params_updateCParamsWhileCompressingState,
) -> ZSTD_compressionParameters {
update_cparams_while_compressing(projection)
}
/// Builds `ZSTD_parameters` with the public default frame parameters.
#[no_mangle]
pub extern "C" fn ZSTD_rust_params_makeParams(
@@ -2481,6 +2602,78 @@ mod tests {
assert_ne!(no_attach, no_row);
}
#[test]
fn mt_update_routes_scalar_policy_and_preserves_saved_window_log() {
let projection = ZSTD_rust_params_updateCParamsWhileCompressingState {
compressionLevel: 3,
cctxSrcSizeHint: 0,
srcSizeHint: ZSTD_CONTENTSIZE_UNKNOWN,
dictSize: 0,
mode: ZSTD_RUST_CPM_NO_ATTACH_DICT,
enableLdm: ZSTD_RUST_PS_DISABLE,
ldmDefaultWindowLog: 27,
overrides: ZSTD_compressionParameters::default(),
useRowMatchFinder: ZSTD_RUST_PS_AUTO,
exclusionMask: 0,
savedWindowLog: 18,
};
let expected = ZSTD_rust_params_getCParamsFromCCtxParams(
projection.compressionLevel,
projection.cctxSrcSizeHint,
projection.srcSizeHint,
projection.dictSize,
projection.mode,
projection.enableLdm,
projection.ldmDefaultWindowLog,
projection.overrides,
projection.useRowMatchFinder,
projection.exclusionMask,
);
let updated = ZSTD_rust_params_updateCParamsWhileCompressing(projection);
assert_eq!(updated.windowLog, projection.savedWindowLog);
assert_eq!(updated.windowLog, 18);
assert_eq!(updated.chainLog, expected.chainLog);
assert_eq!(updated.hashLog, expected.hashLog);
assert_eq!(updated.searchLog, expected.searchLog);
assert_eq!(updated.minMatch, expected.minMatch);
assert_eq!(updated.targetLength, expected.targetLength);
assert_eq!(updated.strategy, expected.strategy);
assert_eq!(ZSTD_rust_params_checkCParams(updated), 0);
}
#[test]
fn mt_update_keeps_exclusion_mask_in_the_rust_routing() {
let exclusion_mask = ZSTD_RUST_EXCLUDE_BTULTRA
| ZSTD_RUST_EXCLUDE_BTOPT
| ZSTD_RUST_EXCLUDE_BTLAZY2
| ZSTD_RUST_EXCLUDE_LAZY2
| ZSTD_RUST_EXCLUDE_LAZY
| ZSTD_RUST_EXCLUDE_GREEDY
| ZSTD_RUST_EXCLUDE_DFAST;
let updated = ZSTD_rust_params_updateCParamsWhileCompressing(
ZSTD_rust_params_updateCParamsWhileCompressingState {
compressionLevel: 3,
cctxSrcSizeHint: 0,
srcSizeHint: ZSTD_CONTENTSIZE_UNKNOWN,
dictSize: 0,
mode: ZSTD_RUST_CPM_NO_ATTACH_DICT,
enableLdm: ZSTD_RUST_PS_DISABLE,
ldmDefaultWindowLog: 27,
overrides: ZSTD_compressionParameters {
strategy: ZSTD_BTULTRA2,
..ZSTD_compressionParameters::default()
},
useRowMatchFinder: ZSTD_RUST_PS_AUTO,
exclusionMask: exclusion_mask,
savedWindowLog: 21,
},
);
assert_eq!(updated.strategy, ZSTD_FAST);
assert_eq!(updated.targetLength, 0);
assert_eq!(updated.windowLog, 21);
assert_eq!(ZSTD_rust_params_checkCParams(updated), 0);
}
#[test]
fn strategy_exclusion_mask_preserves_the_c_fallback_cascade() {
let cparams = policy_cparams(ZSTD_BTULTRA2, 27);