diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index c9a85e6cc..e694a25fd 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -121,11 +121,11 @@ size_t ZSTD_rust_checkBufferStability( /* Context-free compression-parameter selection and sizing leaves live in * Rust (rust/src/zstd_compress_params.rs). This file retains - * configuration-sensitive policy: the excluded-block-compressor strategy - * cascade, private ZSTD_CCtx_params handling, and sanitizer workspace - * policy, which it feeds to the leaves as explicit scalar inputs. The - * ZSTD_CParamMode_e and ZSTD_ParamSwitch_e enums are passed as int; the - * Rust side mirrors their values. */ + * configuration-sensitive policy: the C-preprocessor construction of the + * excluded-block-compressor mask, private ZSTD_CCtx_params handling, and + * sanitizer workspace policy, which it feeds to the leaves as explicit + * scalar inputs. The ZSTD_CParamMode_e and ZSTD_ParamSwitch_e enums are + * passed as int; the Rust side mirrors their values. */ int ZSTD_rust_params_maxCLevel(void); int ZSTD_rust_params_minCLevel(void); int ZSTD_rust_params_defaultCLevel(void); @@ -143,6 +143,13 @@ ZSTD_compressionParameters ZSTD_rust_params_adjustCParams(ZSTD_compressionParameters cParams, U64 srcSize, size_t dictSize, int mode, int useRowMatchFinder); +ZSTD_compressionParameters ZSTD_rust_params_applyStrategyExclusions( + ZSTD_compressionParameters cParams, U32 exclusionMask); +ZSTD_compressionParameters ZSTD_rust_params_getCParamsFromCCtxParams( + int compressionLevel, int cctxSrcSizeHint, U64 srcSizeHint, + size_t dictSize, int mode, int enableLdm, U32 ldmDefaultWindowLog, + ZSTD_compressionParameters overrides, int useRowMatchFinder, + U32 exclusionMask); ZSTD_parameters ZSTD_rust_params_makeParams(ZSTD_compressionParameters cParams); size_t ZSTD_rust_params_maxNbSeq(size_t blockSize, U32 minMatch, int useSequenceProducer); @@ -1141,6 +1148,8 @@ U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) return ZSTD_rust_params_cycleLog(hashLog, (int)strat); } +static U32 ZSTD_getCParamsExclusionMask(void); + /** ZSTD_adjustCParams_internal() : * optimize `cPar` for a specified input (`srcSize` and `dictSize`). * mostly downsize to reduce memory consumption and initialization latency. @@ -1158,46 +1167,10 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, assert(ZSTD_checkCParams(cPar)==0); /* Cascade the selected strategy down to the next-highest one built into - * this binary. */ -#ifdef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR - if (cPar.strategy == ZSTD_btultra2) { - cPar.strategy = ZSTD_btultra; - } - if (cPar.strategy == ZSTD_btultra) { - cPar.strategy = ZSTD_btopt; - } -#endif -#ifdef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR - if (cPar.strategy == ZSTD_btopt) { - cPar.strategy = ZSTD_btlazy2; - } -#endif -#ifdef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR - if (cPar.strategy == ZSTD_btlazy2) { - cPar.strategy = ZSTD_lazy2; - } -#endif -#ifdef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR - if (cPar.strategy == ZSTD_lazy2) { - cPar.strategy = ZSTD_lazy; - } -#endif -#ifdef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR - if (cPar.strategy == ZSTD_lazy) { - cPar.strategy = ZSTD_greedy; - } -#endif -#ifdef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR - if (cPar.strategy == ZSTD_greedy) { - cPar.strategy = ZSTD_dfast; - } -#endif -#ifdef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR - if (cPar.strategy == ZSTD_dfast) { - cPar.strategy = ZSTD_fast; - cPar.targetLength = 0; - } -#endif + * this binary. The C preprocessor constructs the mask so Rust does not + * need to mirror this build's active ZSTD_EXCLUDE_* definitions. */ + cPar = ZSTD_rust_params_applyStrategyExclusions( + cPar, ZSTD_getCParamsExclusionMask()); /* The remaining adjustment logic is context-free and lives in Rust. The * short-cache and row-hash tag widths the leaf assumes are fixed @@ -1221,20 +1194,53 @@ ZSTD_adjustCParams(ZSTD_compressionParameters cPar, static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode); static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode); +enum { + ZSTD_RUST_PARAMS_EXCLUDE_BTULTRA = 1u << 0, + ZSTD_RUST_PARAMS_EXCLUDE_BTOPT = 1u << 1, + ZSTD_RUST_PARAMS_EXCLUDE_BTLAZY2 = 1u << 2, + ZSTD_RUST_PARAMS_EXCLUDE_LAZY2 = 1u << 3, + ZSTD_RUST_PARAMS_EXCLUDE_LAZY = 1u << 4, + ZSTD_RUST_PARAMS_EXCLUDE_GREEDY = 1u << 5, + ZSTD_RUST_PARAMS_EXCLUDE_DFAST = 1u << 6 +}; + +static U32 ZSTD_getCParamsExclusionMask(void) +{ + U32 mask = 0; +#ifdef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR + mask |= ZSTD_RUST_PARAMS_EXCLUDE_BTULTRA; +#endif +#ifdef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR + mask |= ZSTD_RUST_PARAMS_EXCLUDE_BTOPT; +#endif +#ifdef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR + mask |= ZSTD_RUST_PARAMS_EXCLUDE_BTLAZY2; +#endif +#ifdef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR + mask |= ZSTD_RUST_PARAMS_EXCLUDE_LAZY2; +#endif +#ifdef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR + mask |= ZSTD_RUST_PARAMS_EXCLUDE_LAZY; +#endif +#ifdef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR + mask |= ZSTD_RUST_PARAMS_EXCLUDE_GREEDY; +#endif +#ifdef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR + mask |= ZSTD_RUST_PARAMS_EXCLUDE_DFAST; +#endif + return mask; +} + ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode) { - ZSTD_compressionParameters cParams; - if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) { - assert(CCtxParams->srcSizeHint>=0); - srcSizeHint = (U64)CCtxParams->srcSizeHint; - } - cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize, mode); - if (CCtxParams->ldmParams.enableLdm == ZSTD_ps_enable) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; - ZSTD_rust_params_overrideCParams(&cParams, &CCtxParams->cParams); - assert(!ZSTD_checkCParams(cParams)); - /* srcSizeHint == 0 means 0 */ - return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode, CCtxParams->useRowMatchFinder); + return ZSTD_rust_params_getCParamsFromCCtxParams( + CCtxParams->compressionLevel, CCtxParams->srcSizeHint, + srcSizeHint, dictSize, (int)mode, + (int)CCtxParams->ldmParams.enableLdm, + ZSTD_LDM_DEFAULT_WINDOW_LOG, CCtxParams->cParams, + (int)CCtxParams->useRowMatchFinder, + ZSTD_getCParamsExclusionMask()); } static size_t diff --git a/rust/src/zstd_compress_params.rs b/rust/src/zstd_compress_params.rs index 0cf067a76..dd0126335 100644 --- a/rust/src/zstd_compress_params.rs +++ b/rust/src/zstd_compress_params.rs @@ -6,13 +6,12 @@ //! Context-free compression-parameter selection and sizing leaves. //! //! This module deliberately does **not** own the public `ZSTD_*` symbols yet. -//! `zstd_compress.c` still owns configuration-sensitive policy: excluded block -//! compressors, private `ZSTD_CCtx_params` layouts, LDM workspace sizing, and -//! ASAN workspace policy. The C integration layer can select a raw table -//! entry here, apply its configured strategy cascade, then use the adjustment -//! and sizing leaves below. That keeps the Rust implementation independent of -//! C preprocessor state while retaining byte-for-byte C policy for reduced -//! builds. +//! `zstd_compress.c` still owns configuration-sensitive policy: private +//! `ZSTD_CCtx_params` layouts, the C-preprocessor construction of excluded +//! block-compressor bits, LDM workspace sizing, and ASAN workspace policy. +//! The Rust policy leaves receive those build values as explicit scalar inputs, +//! retaining byte-for-byte C behavior for reduced builds without exposing +//! private C state across the ABI. use crate::errors::{ZstdErrorCode, ERROR}; use std::mem::size_of; @@ -54,6 +53,17 @@ const ZSTD_BTOPT: c_int = 7; const ZSTD_BTULTRA: c_int = 8; const ZSTD_BTULTRA2: c_int = 9; +/* Build-policy bits supplied by the C shim. C preprocessor configuration + * stays on the C side; the bit positions are the ABI between the shim and + * this pure strategy cascade. */ +const ZSTD_RUST_EXCLUDE_BTULTRA: u32 = 1 << 0; +const ZSTD_RUST_EXCLUDE_BTOPT: u32 = 1 << 1; +const ZSTD_RUST_EXCLUDE_BTLAZY2: u32 = 1 << 2; +const ZSTD_RUST_EXCLUDE_LAZY2: u32 = 1 << 3; +const ZSTD_RUST_EXCLUDE_LAZY: u32 = 1 << 4; +const ZSTD_RUST_EXCLUDE_GREEDY: u32 = 1 << 5; +const ZSTD_RUST_EXCLUDE_DFAST: u32 = 1 << 6; + const ZSTD_DICT_FORCE_ATTACH: c_int = 1; const ZSTD_DICT_FORCE_COPY: c_int = 2; @@ -760,6 +770,41 @@ fn adjust_cparams( cparams } +#[inline] +fn apply_strategy_exclusions( + mut cparams: ZSTD_compressionParameters, + exclusion_mask: u32, +) -> ZSTD_compressionParameters { + if exclusion_mask & ZSTD_RUST_EXCLUDE_BTULTRA != 0 { + if cparams.strategy == ZSTD_BTULTRA2 { + cparams.strategy = ZSTD_BTULTRA; + } + if cparams.strategy == ZSTD_BTULTRA { + cparams.strategy = ZSTD_BTOPT; + } + } + if exclusion_mask & ZSTD_RUST_EXCLUDE_BTOPT != 0 && cparams.strategy == ZSTD_BTOPT { + cparams.strategy = ZSTD_BTLAZY2; + } + if exclusion_mask & ZSTD_RUST_EXCLUDE_BTLAZY2 != 0 && cparams.strategy == ZSTD_BTLAZY2 { + cparams.strategy = ZSTD_LAZY2; + } + if exclusion_mask & ZSTD_RUST_EXCLUDE_LAZY2 != 0 && cparams.strategy == ZSTD_LAZY2 { + cparams.strategy = ZSTD_LAZY; + } + if exclusion_mask & ZSTD_RUST_EXCLUDE_LAZY != 0 && cparams.strategy == ZSTD_LAZY { + cparams.strategy = ZSTD_GREEDY; + } + if exclusion_mask & ZSTD_RUST_EXCLUDE_GREEDY != 0 && cparams.strategy == ZSTD_GREEDY { + cparams.strategy = ZSTD_DFAST; + } + if exclusion_mask & ZSTD_RUST_EXCLUDE_DFAST != 0 && cparams.strategy == ZSTD_DFAST { + cparams.strategy = ZSTD_FAST; + cparams.targetLength = 0; + } + cparams +} + #[inline] fn get_cparam_row_size(src_size_hint: u64, dict_size: usize, mode: c_int) -> u64 { let mut dict_size = dict_size as u64; @@ -782,8 +827,8 @@ fn get_cparam_row_size(src_size_hint: u64, dict_size: usize, mode: c_int) -> u64 /// Selects a raw compression-level-table entry, without strategy cascading or /// source/dictionary adjustment. /// -/// A C wrapper must apply its active excluded-compressor cascade to the -/// returned strategy before calling [`ZSTD_rust_params_adjustCParams`]. +/// The C wrapper supplies its active excluded-compressor configuration as a +/// mask; the orchestration leaf applies it before each adjustment stage. fn select_cparams( compression_level: c_int, src_size_hint: u64, @@ -810,6 +855,51 @@ fn select_cparams( cparams } +fn get_cparams_from_cctx_params( + compression_level: c_int, + cctx_src_size_hint: c_int, + mut src_size_hint: u64, + dict_size: usize, + mode: c_int, + enable_ldm: c_int, + ldm_default_window_log: u32, + overrides: ZSTD_compressionParameters, + use_row_match_finder: c_int, + exclusion_mask: u32, +) -> ZSTD_compressionParameters { + if src_size_hint == ZSTD_CONTENTSIZE_UNKNOWN && cctx_src_size_hint > 0 { + debug_assert!(cctx_src_size_hint >= 0); + src_size_hint = cctx_src_size_hint as u64; + } + + /* ZSTD_getCParams_internal() selects and performs the first adjustment. */ + let mut cparams = adjust_cparams( + apply_strategy_exclusions( + select_cparams(compression_level, src_size_hint, dict_size, mode), + exclusion_mask, + ), + src_size_hint, + dict_size, + mode, + ZSTD_RUST_PS_AUTO, + ); + + if enable_ldm == ZSTD_RUST_PS_ENABLE { + cparams.windowLog = ldm_default_window_log; + } + override_cparams(&mut cparams, &overrides); + + /* Keep the C-side assertion immediately before the final adjustment. */ + debug_assert_eq!(check_cparams(cparams), 0); + adjust_cparams( + apply_strategy_exclusions(cparams, exclusion_mask), + src_size_hint, + dict_size, + mode, + use_row_match_finder, + ) +} + #[inline] fn make_params(cparams: ZSTD_compressionParameters) -> ZSTD_parameters { ZSTD_parameters { @@ -919,6 +1009,15 @@ pub extern "C" fn ZSTD_rust_params_getCParamRowSize( get_cparam_row_size(srcSizeHint, dictSize, mode) } +/// Applies the C-selected build's excluded block-compressor cascade. +#[no_mangle] +pub extern "C" fn ZSTD_rust_params_applyStrategyExclusions( + cparams: ZSTD_compressionParameters, + exclusionMask: u32, +) -> ZSTD_compressionParameters { + apply_strategy_exclusions(cparams, exclusionMask) +} + /// Returns the unadjusted compression-level-table entry. #[no_mangle] pub extern "C" fn ZSTD_rust_params_selectCParams( @@ -944,6 +1043,38 @@ pub extern "C" fn ZSTD_rust_params_adjustCParams( adjust_cparams(cparams, srcSize, dictSize, mode, useRowMatchFinder) } +/// Reproduces `ZSTD_getCParamsFromCCtxParams()` from scalar snapshots. +/// +/// The C shim supplies the private build policy as an exclusion mask and the +/// LDM default window log, while the context-free selection, override, and +/// two adjustment stages remain entirely in Rust. +#[no_mangle] +pub extern "C" fn ZSTD_rust_params_getCParamsFromCCtxParams( + compressionLevel: c_int, + cctxSrcSizeHint: c_int, + srcSizeHint: u64, + dictSize: usize, + mode: c_int, + enableLdm: c_int, + ldmDefaultWindowLog: u32, + overrides: ZSTD_compressionParameters, + useRowMatchFinder: c_int, + exclusionMask: u32, +) -> ZSTD_compressionParameters { + get_cparams_from_cctx_params( + compressionLevel, + cctxSrcSizeHint, + srcSizeHint, + dictSize, + mode, + enableLdm, + ldmDefaultWindowLog, + overrides, + useRowMatchFinder, + exclusionMask, + ) +} + /// Builds `ZSTD_parameters` with the public default frame parameters. #[no_mangle] pub extern "C" fn ZSTD_rust_params_makeParams( @@ -1944,6 +2075,202 @@ mod tests { ); } + fn cctx_policy_params( + cctx_src_size_hint: c_int, + src_size_hint: u64, + dict_size: usize, + mode: c_int, + enable_ldm: c_int, + overrides: ZSTD_compressionParameters, + use_row_match_finder: c_int, + ) -> ZSTD_compressionParameters { + get_cparams_from_cctx_params( + 3, + cctx_src_size_hint, + src_size_hint, + dict_size, + mode, + enable_ldm, + 27, + overrides, + use_row_match_finder, + 0, + ) + } + + #[test] + fn cctx_source_size_hint_preserves_unknown_and_zero_semantics() { + let unknown = cctx_policy_params( + 0, + ZSTD_CONTENTSIZE_UNKNOWN, + 0, + ZSTD_RUST_CPM_UNKNOWN, + ZSTD_RUST_PS_DISABLE, + ZSTD_compressionParameters::default(), + ZSTD_RUST_PS_AUTO, + ); + let hinted = cctx_policy_params( + 16 * 1024, + ZSTD_CONTENTSIZE_UNKNOWN, + 0, + ZSTD_RUST_CPM_UNKNOWN, + ZSTD_RUST_PS_DISABLE, + ZSTD_compressionParameters::default(), + ZSTD_RUST_PS_AUTO, + ); + let explicitly_sized = cctx_policy_params( + 0, + 16 * 1024, + 0, + ZSTD_RUST_CPM_UNKNOWN, + ZSTD_RUST_PS_DISABLE, + ZSTD_compressionParameters::default(), + ZSTD_RUST_PS_AUTO, + ); + assert_eq!(hinted, explicitly_sized); + assert!(unknown.windowLog > hinted.windowLog); + + let known_zero = cctx_policy_params( + 0, + 0, + 0, + ZSTD_RUST_CPM_UNKNOWN, + ZSTD_RUST_PS_DISABLE, + ZSTD_compressionParameters::default(), + ZSTD_RUST_PS_AUTO, + ); + let zero_with_hint = cctx_policy_params( + 16 * 1024, + 0, + 0, + ZSTD_RUST_CPM_UNKNOWN, + ZSTD_RUST_PS_DISABLE, + ZSTD_compressionParameters::default(), + ZSTD_RUST_PS_AUTO, + ); + assert_eq!(known_zero, zero_with_hint); + assert!(hinted.windowLog > known_zero.windowLog); + } + + #[test] + fn cctx_ldm_override_uses_the_explicit_default_window_log() { + let normal = cctx_policy_params( + 0, + ZSTD_CONTENTSIZE_UNKNOWN, + 0, + ZSTD_RUST_CPM_UNKNOWN, + ZSTD_RUST_PS_DISABLE, + ZSTD_compressionParameters::default(), + ZSTD_RUST_PS_AUTO, + ); + let ldm = cctx_policy_params( + 0, + ZSTD_CONTENTSIZE_UNKNOWN, + 0, + ZSTD_RUST_CPM_UNKNOWN, + ZSTD_RUST_PS_ENABLE, + ZSTD_compressionParameters::default(), + ZSTD_RUST_PS_AUTO, + ); + assert_eq!(ldm.windowLog, 27); + assert_ne!(normal.windowLog, ldm.windowLog); + } + + #[test] + fn cctx_nonzero_overrides_replace_only_selected_fields() { + let baseline = cctx_policy_params( + 0, + ZSTD_CONTENTSIZE_UNKNOWN, + 0, + ZSTD_RUST_CPM_UNKNOWN, + ZSTD_RUST_PS_DISABLE, + ZSTD_compressionParameters::default(), + ZSTD_RUST_PS_DISABLE, + ); + let result = cctx_policy_params( + 0, + ZSTD_CONTENTSIZE_UNKNOWN, + 0, + ZSTD_RUST_CPM_UNKNOWN, + ZSTD_RUST_PS_ENABLE, + ZSTD_compressionParameters { + windowLog: 12, + chainLog: 0, + hashLog: 29, + searchLog: 4, + minMatch: 6, + targetLength: 0, + strategy: ZSTD_GREEDY, + }, + ZSTD_RUST_PS_DISABLE, + ); + assert_eq!(result.windowLog, 12); + assert_eq!(result.hashLog, 29); + assert_eq!(result.searchLog, 4); + assert_eq!(result.minMatch, 6); + assert_eq!(result.strategy, ZSTD_GREEDY); + assert_eq!(result.chainLog, baseline.chainLog); + assert_eq!(result.targetLength, baseline.targetLength); + } + + #[test] + fn cctx_mode_and_row_match_finder_flow_reaches_final_adjustment() { + let overrides = ZSTD_compressionParameters { + hashLog: 30, + searchLog: 4, + strategy: ZSTD_GREEDY, + ..ZSTD_compressionParameters::default() + }; + let no_row = cctx_policy_params( + 0, + ZSTD_CONTENTSIZE_UNKNOWN, + 32 * 1024, + ZSTD_RUST_CPM_ATTACH_DICT, + ZSTD_RUST_PS_DISABLE, + overrides, + ZSTD_RUST_PS_DISABLE, + ); + let row = cctx_policy_params( + 0, + ZSTD_CONTENTSIZE_UNKNOWN, + 32 * 1024, + ZSTD_RUST_CPM_ATTACH_DICT, + ZSTD_RUST_PS_DISABLE, + overrides, + ZSTD_RUST_PS_ENABLE, + ); + let no_attach = cctx_policy_params( + 0, + ZSTD_CONTENTSIZE_UNKNOWN, + 32 * 1024, + ZSTD_RUST_CPM_NO_ATTACH_DICT, + ZSTD_RUST_PS_DISABLE, + overrides, + ZSTD_RUST_PS_DISABLE, + ); + assert_eq!(no_row.hashLog, 30); + assert_eq!(row.hashLog, 28); + assert_ne!(no_attach, no_row); + } + + #[test] + fn strategy_exclusion_mask_preserves_the_c_fallback_cascade() { + let cparams = policy_cparams(ZSTD_BTULTRA2, 27); + let all_excluded = ZSTD_RUST_EXCLUDE_BTULTRA + | ZSTD_RUST_EXCLUDE_BTOPT + | ZSTD_RUST_EXCLUDE_BTLAZY2 + | ZSTD_RUST_EXCLUDE_LAZY2 + | ZSTD_RUST_EXCLUDE_LAZY + | ZSTD_RUST_EXCLUDE_GREEDY + | ZSTD_RUST_EXCLUDE_DFAST; + let cascaded = apply_strategy_exclusions(cparams, all_excluded); + assert_eq!(cascaded.strategy, ZSTD_FAST); + assert_eq!(cascaded.targetLength, 0); + + let first_only = apply_strategy_exclusions(cparams, ZSTD_RUST_EXCLUDE_BTULTRA); + assert_eq!(first_only.strategy, ZSTD_BTOPT); + } + #[test] fn adjustment_clamps_and_downsizes_like_the_c_leaf() { let input = ZSTD_compressionParameters {