refactor(compress): move MT parameter update policy to Rust
Move the next-job parameter derivation in ZSTDMT_updateCParams_whileCompressing() behind a scalar Rust projection. Rust now reproduces the existing CCtx-parameter derivation and restores the active frame's saved windowLog; C retains private MT/CCtx layouts, logging, and final writes. Expose the preprocessor-derived block-compressor exclusion mask through the private C boundary so reduced builds keep their authoritative configuration. Add C/Rust layout assertions and focused tests for window preservation and strategy-exclusion routing. Test Plan: - `ulimit -v 41943040; cargo +nightly fmt --manifest-path rust/Cargo.toml --all -- --check` - `ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings` - `ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings` - `ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/cli/Cargo.toml --all-targets` (184 tests) - `ulimit -v 41943040; make -j1` - `ulimit -v 41943040; make -j1 -C tests test` (all upstream tests completed successfully) - Standalone Rust unit linking remains unavailable without the C-owned bridge symbols; integrated build and upstream MT/API coverage passed.
This commit is contained in:
@@ -15,7 +15,7 @@
|
||||
//! state across the ABI.
|
||||
|
||||
use crate::errors::{ZstdErrorCode, ERROR};
|
||||
use std::mem::size_of;
|
||||
use std::mem::{offset_of, size_of};
|
||||
use std::os::raw::c_int;
|
||||
|
||||
pub const ZSTD_CONTENTSIZE_UNKNOWN: u64 = u64::MAX;
|
||||
@@ -130,6 +130,98 @@ pub struct ZSTD_bounds {
|
||||
pub upperBound: c_int,
|
||||
}
|
||||
|
||||
/// Scalar snapshot used when MT compression updates parameters between jobs.
|
||||
///
|
||||
/// The private `ZSTD_CCtx_params` and `ZSTDMT_CCtx` layouts stay in C. This
|
||||
/// projection carries the same scalar inputs as
|
||||
/// `ZSTD_getCParamsFromCCtxParams()` and the active frame's window log.
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub struct ZSTD_rust_params_updateCParamsWhileCompressingState {
|
||||
pub compressionLevel: c_int,
|
||||
pub cctxSrcSizeHint: c_int,
|
||||
pub srcSizeHint: u64,
|
||||
pub dictSize: usize,
|
||||
pub mode: c_int,
|
||||
pub enableLdm: c_int,
|
||||
pub ldmDefaultWindowLog: u32,
|
||||
pub overrides: ZSTD_compressionParameters,
|
||||
pub useRowMatchFinder: c_int,
|
||||
pub exclusionMask: u32,
|
||||
pub savedWindowLog: u32,
|
||||
}
|
||||
|
||||
const _: () = {
|
||||
assert!(
|
||||
offset_of!(
|
||||
ZSTD_rust_params_updateCParamsWhileCompressingState,
|
||||
compressionLevel
|
||||
) == 0
|
||||
);
|
||||
assert!(
|
||||
offset_of!(
|
||||
ZSTD_rust_params_updateCParamsWhileCompressingState,
|
||||
cctxSrcSizeHint
|
||||
) == size_of::<c_int>()
|
||||
);
|
||||
assert!(
|
||||
offset_of!(
|
||||
ZSTD_rust_params_updateCParamsWhileCompressingState,
|
||||
srcSizeHint
|
||||
) == 2 * size_of::<c_int>()
|
||||
);
|
||||
assert!(
|
||||
offset_of!(
|
||||
ZSTD_rust_params_updateCParamsWhileCompressingState,
|
||||
dictSize
|
||||
) == 16
|
||||
);
|
||||
assert!(
|
||||
offset_of!(ZSTD_rust_params_updateCParamsWhileCompressingState, mode)
|
||||
== if size_of::<usize>() == 8 { 24 } else { 20 }
|
||||
);
|
||||
assert!(
|
||||
offset_of!(
|
||||
ZSTD_rust_params_updateCParamsWhileCompressingState,
|
||||
enableLdm
|
||||
) == if size_of::<usize>() == 8 { 28 } else { 24 }
|
||||
);
|
||||
assert!(
|
||||
offset_of!(
|
||||
ZSTD_rust_params_updateCParamsWhileCompressingState,
|
||||
ldmDefaultWindowLog
|
||||
) == if size_of::<usize>() == 8 { 32 } else { 28 }
|
||||
);
|
||||
assert!(
|
||||
offset_of!(
|
||||
ZSTD_rust_params_updateCParamsWhileCompressingState,
|
||||
overrides
|
||||
) == if size_of::<usize>() == 8 { 36 } else { 32 }
|
||||
);
|
||||
assert!(
|
||||
offset_of!(
|
||||
ZSTD_rust_params_updateCParamsWhileCompressingState,
|
||||
useRowMatchFinder
|
||||
) == if size_of::<usize>() == 8 { 64 } else { 60 }
|
||||
);
|
||||
assert!(
|
||||
offset_of!(
|
||||
ZSTD_rust_params_updateCParamsWhileCompressingState,
|
||||
exclusionMask
|
||||
) == if size_of::<usize>() == 8 { 68 } else { 64 }
|
||||
);
|
||||
assert!(
|
||||
offset_of!(
|
||||
ZSTD_rust_params_updateCParamsWhileCompressingState,
|
||||
savedWindowLog
|
||||
) == if size_of::<usize>() == 8 { 72 } else { 68 }
|
||||
);
|
||||
assert!(
|
||||
size_of::<ZSTD_rust_params_updateCParamsWhileCompressingState>()
|
||||
== if size_of::<usize>() == 8 { 80 } else { 72 }
|
||||
);
|
||||
};
|
||||
|
||||
/* `clevels.h`, represented as raw fields so the table stays compact and easy
|
||||
* to compare mechanically against its C source. Field order is W, C, H, S,
|
||||
* L, TL, strategy. */
|
||||
@@ -969,6 +1061,25 @@ fn get_cparams_from_cctx_params(
|
||||
)
|
||||
}
|
||||
|
||||
fn update_cparams_while_compressing(
|
||||
projection: ZSTD_rust_params_updateCParamsWhileCompressingState,
|
||||
) -> ZSTD_compressionParameters {
|
||||
let mut cparams = get_cparams_from_cctx_params(
|
||||
projection.compressionLevel,
|
||||
projection.cctxSrcSizeHint,
|
||||
projection.srcSizeHint,
|
||||
projection.dictSize,
|
||||
projection.mode,
|
||||
projection.enableLdm,
|
||||
projection.ldmDefaultWindowLog,
|
||||
projection.overrides,
|
||||
projection.useRowMatchFinder,
|
||||
projection.exclusionMask,
|
||||
);
|
||||
cparams.windowLog = projection.savedWindowLog;
|
||||
cparams
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn make_params(cparams: ZSTD_compressionParameters) -> ZSTD_parameters {
|
||||
ZSTD_parameters {
|
||||
@@ -1195,6 +1306,16 @@ pub extern "C" fn ZSTD_rust_params_getCParamsFromCCtxParams(
|
||||
)
|
||||
}
|
||||
|
||||
/// Derives the next MT job's parameters while preserving the active frame's
|
||||
/// window log. The C shim owns the private-context projection and final
|
||||
/// writes; Rust owns the parameter derivation and update policy.
|
||||
#[no_mangle]
|
||||
pub extern "C" fn ZSTD_rust_params_updateCParamsWhileCompressing(
|
||||
projection: ZSTD_rust_params_updateCParamsWhileCompressingState,
|
||||
) -> ZSTD_compressionParameters {
|
||||
update_cparams_while_compressing(projection)
|
||||
}
|
||||
|
||||
/// Builds `ZSTD_parameters` with the public default frame parameters.
|
||||
#[no_mangle]
|
||||
pub extern "C" fn ZSTD_rust_params_makeParams(
|
||||
@@ -2481,6 +2602,78 @@ mod tests {
|
||||
assert_ne!(no_attach, no_row);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mt_update_routes_scalar_policy_and_preserves_saved_window_log() {
|
||||
let projection = ZSTD_rust_params_updateCParamsWhileCompressingState {
|
||||
compressionLevel: 3,
|
||||
cctxSrcSizeHint: 0,
|
||||
srcSizeHint: ZSTD_CONTENTSIZE_UNKNOWN,
|
||||
dictSize: 0,
|
||||
mode: ZSTD_RUST_CPM_NO_ATTACH_DICT,
|
||||
enableLdm: ZSTD_RUST_PS_DISABLE,
|
||||
ldmDefaultWindowLog: 27,
|
||||
overrides: ZSTD_compressionParameters::default(),
|
||||
useRowMatchFinder: ZSTD_RUST_PS_AUTO,
|
||||
exclusionMask: 0,
|
||||
savedWindowLog: 18,
|
||||
};
|
||||
let expected = ZSTD_rust_params_getCParamsFromCCtxParams(
|
||||
projection.compressionLevel,
|
||||
projection.cctxSrcSizeHint,
|
||||
projection.srcSizeHint,
|
||||
projection.dictSize,
|
||||
projection.mode,
|
||||
projection.enableLdm,
|
||||
projection.ldmDefaultWindowLog,
|
||||
projection.overrides,
|
||||
projection.useRowMatchFinder,
|
||||
projection.exclusionMask,
|
||||
);
|
||||
let updated = ZSTD_rust_params_updateCParamsWhileCompressing(projection);
|
||||
assert_eq!(updated.windowLog, projection.savedWindowLog);
|
||||
assert_eq!(updated.windowLog, 18);
|
||||
assert_eq!(updated.chainLog, expected.chainLog);
|
||||
assert_eq!(updated.hashLog, expected.hashLog);
|
||||
assert_eq!(updated.searchLog, expected.searchLog);
|
||||
assert_eq!(updated.minMatch, expected.minMatch);
|
||||
assert_eq!(updated.targetLength, expected.targetLength);
|
||||
assert_eq!(updated.strategy, expected.strategy);
|
||||
assert_eq!(ZSTD_rust_params_checkCParams(updated), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mt_update_keeps_exclusion_mask_in_the_rust_routing() {
|
||||
let exclusion_mask = ZSTD_RUST_EXCLUDE_BTULTRA
|
||||
| ZSTD_RUST_EXCLUDE_BTOPT
|
||||
| ZSTD_RUST_EXCLUDE_BTLAZY2
|
||||
| ZSTD_RUST_EXCLUDE_LAZY2
|
||||
| ZSTD_RUST_EXCLUDE_LAZY
|
||||
| ZSTD_RUST_EXCLUDE_GREEDY
|
||||
| ZSTD_RUST_EXCLUDE_DFAST;
|
||||
let updated = ZSTD_rust_params_updateCParamsWhileCompressing(
|
||||
ZSTD_rust_params_updateCParamsWhileCompressingState {
|
||||
compressionLevel: 3,
|
||||
cctxSrcSizeHint: 0,
|
||||
srcSizeHint: ZSTD_CONTENTSIZE_UNKNOWN,
|
||||
dictSize: 0,
|
||||
mode: ZSTD_RUST_CPM_NO_ATTACH_DICT,
|
||||
enableLdm: ZSTD_RUST_PS_DISABLE,
|
||||
ldmDefaultWindowLog: 27,
|
||||
overrides: ZSTD_compressionParameters {
|
||||
strategy: ZSTD_BTULTRA2,
|
||||
..ZSTD_compressionParameters::default()
|
||||
},
|
||||
useRowMatchFinder: ZSTD_RUST_PS_AUTO,
|
||||
exclusionMask: exclusion_mask,
|
||||
savedWindowLog: 21,
|
||||
},
|
||||
);
|
||||
assert_eq!(updated.strategy, ZSTD_FAST);
|
||||
assert_eq!(updated.targetLength, 0);
|
||||
assert_eq!(updated.windowLog, 21);
|
||||
assert_eq!(ZSTD_rust_params_checkCParams(updated), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strategy_exclusion_mask_preserves_the_c_fallback_cascade() {
|
||||
let cparams = policy_cparams(ZSTD_BTULTRA2, 27);
|
||||
|
||||
Reference in New Issue
Block a user