feat(compress): move MT sizing policy to Rust

Move target job-log, overlap-log, and overlap-size policy into the Rust multithreaded compression module, retaining the C context projection and logging wrapper. Add policy tests for LDM, strategy defaults, bounds, and overlap sizing.

Test Plan: cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression; cargo clippy --manifest-path rust/Cargo.toml; cargo clippy --manifest-path rust/Cargo.toml --benches; cargo clippy --manifest-path rust/Cargo.toml --tests; make -B -C lib -j2 lib; make -B -C programs -j2 zstd zstd-small zstd-frugal; make -B -C tests -j2 test-cli-tests; make -B -C tests -j2 test-zstream

rust/src/zstdmt_compress.rs

@@ -7,16 +7,138 @@
 //!
 //! The serial LDM state, job descriptor fields, worker callback, and streaming
 //! state still use private C layouts.  `zstdmt_compress.c` therefore keeps
-//! those operations and projects only allocation/lifecycle pieces into this
-//! module.  The entry points below are narrow C ABIs: buffers, `ZSTD_CCtx *`
-//! values, and job descriptors remain opaque to Rust, while allocation, reuse,
-//! expansion, and synchronization of Rust-owned state are Rust-owned.
+//! those operations and projects only allocation/lifecycle pieces and pure
+//! sizing policy into this module.  The entry points below are narrow C ABIs:
+//! buffers, `ZSTD_CCtx *` values, and job descriptors remain opaque to Rust,
+//! while allocation, reuse, expansion, synchronization, and sizing policy are
+//! Rust-owned.
 
 use std::mem::{self, MaybeUninit};
-use std::os::raw::{c_uint, c_void};
+use std::os::raw::{c_int, c_uint, c_void};
 use std::ptr;
 use std::sync::Mutex;
 
+const ZSTDMT_JOBLOG_MAX: c_uint = if mem::size_of::<usize>() == 4 { 29 } else { 30 };
+const ZSTD_WINDOWLOG_MAX: c_uint = if mem::size_of::<usize>() == 4 { 30 } else { 31 };
+
+const ZSTD_FAST: c_int = 1;
+const ZSTD_DFAST: c_int = 2;
+const ZSTD_GREEDY: c_int = 3;
+const ZSTD_LAZY: c_int = 4;
+const ZSTD_LAZY2: c_int = 5;
+const ZSTD_BTLAZY2: c_int = 6;
+const ZSTD_BTOPT: c_int = 7;
+const ZSTD_BTULTRA: c_int = 8;
+const ZSTD_BTULTRA2: c_int = 9;
+const ZSTD_PS_ENABLE: c_int = 1;
+#[cfg(test)]
+const ZSTD_PS_DISABLE: c_int = 2;
+
+#[inline]
+fn cycle_log(chain_log: c_uint, strategy: c_int) -> c_uint {
+    chain_log.wrapping_sub((strategy >= ZSTD_BTLAZY2) as c_uint)
+}
+
+#[inline]
+fn compute_target_job_log(
+    window_log: c_uint,
+    chain_log: c_uint,
+    strategy: c_int,
+    enable_ldm: c_int,
+) -> c_uint {
+    let job_log = if enable_ldm == ZSTD_PS_ENABLE {
+        /* In Long Range Mode, the windowLog is typically oversized.
+         * In which case, it's preferable to determine the jobSize
+         * based on cycleLog instead. */
+        21.max(cycle_log(chain_log, strategy).wrapping_add(3))
+    } else {
+        20.max(window_log.wrapping_add(2))
+    };
+    job_log.min(ZSTDMT_JOBLOG_MAX)
+}
+
+#[inline]
+fn overlap_log_default(strategy: c_int) -> c_int {
+    match strategy {
+        ZSTD_BTULTRA2 => 9,
+        ZSTD_BTULTRA | ZSTD_BTOPT => 8,
+        ZSTD_BTLAZY2 | ZSTD_LAZY2 => 7,
+        ZSTD_LAZY | ZSTD_GREEDY | ZSTD_DFAST | ZSTD_FAST => 6,
+        _ => 6,
+    }
+}
+
+#[inline]
+fn overlap_log(overlap_log: c_int, strategy: c_int) -> c_int {
+    debug_assert!((0..=9).contains(&overlap_log));
+    if overlap_log == 0 {
+        overlap_log_default(strategy)
+    } else {
+        overlap_log
+    }
+}
+
+#[inline]
+fn compute_overlap_size(
+    window_log: c_uint,
+    chain_log: c_uint,
+    strategy: c_int,
+    overlap_log_value: c_int,
+    enable_ldm: c_int,
+) -> usize {
+    let overlap_r_log = 9 - overlap_log(overlap_log_value, strategy);
+    let mut overlap_log_value = if overlap_r_log >= 8 {
+        0
+    } else {
+        window_log as c_int - overlap_r_log
+    };
+    debug_assert!((0..=8).contains(&overlap_r_log));
+    if enable_ldm == ZSTD_PS_ENABLE {
+        /* In Long Range Mode, the windowLog is typically oversized.
+         * In which case, it's preferable to determine the jobSize
+         * based on chainLog instead.
+         * Then, ovLog becomes a fraction of the jobSize, rather than windowSize */
+        let target_job_log = compute_target_job_log(window_log, chain_log, strategy, enable_ldm);
+        overlap_log_value = window_log.min(target_job_log.wrapping_sub(2)) as c_int - overlap_r_log;
+    }
+    debug_assert!(overlap_log_value >= 0);
+    debug_assert!(overlap_log_value <= ZSTD_WINDOWLOG_MAX as c_int);
+    if overlap_log_value == 0 {
+        0
+    } else {
+        1usize << overlap_log_value as usize
+    }
+}
+
+/// C ABI for the pure MT target job-log policy.
+#[no_mangle]
+pub extern "C" fn ZSTDMT_rust_computeTargetJobLog(
+    windowLog: c_uint,
+    chainLog: c_uint,
+    strategy: c_int,
+    enableLdm: c_int,
+) -> c_uint {
+    compute_target_job_log(windowLog, chainLog, strategy, enableLdm)
+}
+
+/// C ABI for the MT overlap-log default/selection policy.
+#[no_mangle]
+pub extern "C" fn ZSTDMT_rust_overlapLog(overlapLog: c_int, strategy: c_int) -> c_int {
+    overlap_log(overlapLog, strategy)
+}
+
+/// C ABI for the pure MT overlap-size policy.
+#[no_mangle]
+pub extern "C" fn ZSTDMT_rust_computeOverlapSize(
+    windowLog: c_uint,
+    chainLog: c_uint,
+    strategy: c_int,
+    overlapLog: c_int,
+    enableLdm: c_int,
+) -> usize {
+    compute_overlap_size(windowLog, chainLog, strategy, overlapLog, enableLdm)
+}
+
 type ZstdAllocFunction = unsafe extern "C" fn(*mut c_void, usize) -> *mut c_void;
 type ZstdFreeFunction = unsafe extern "C" fn(*mut c_void, *mut c_void);
 
@@ -680,4 +802,71 @@ mod tests {
         assert_eq!(rounded_job_count(255), Some(256));
         assert_eq!(rounded_job_count(256), Some(512));
     }
+
+    #[test]
+    fn target_job_log_preserves_ldm_and_non_ldm_policy() {
+        assert_eq!(
+            compute_target_job_log(18, 25, ZSTD_FAST, ZSTD_PS_DISABLE),
+            20
+        );
+        assert_eq!(
+            compute_target_job_log(30, 25, ZSTD_FAST, ZSTD_PS_DISABLE),
+            ZSTDMT_JOBLOG_MAX
+        );
+        assert_eq!(
+            compute_target_job_log(10, 25, ZSTD_FAST, ZSTD_PS_ENABLE),
+            28
+        );
+        assert_eq!(
+            compute_target_job_log(30, 25, ZSTD_BTULTRA2, ZSTD_PS_ENABLE),
+            27
+        );
+        assert_eq!(
+            compute_target_job_log(10, 30, ZSTD_BTULTRA2, ZSTD_PS_ENABLE),
+            ZSTDMT_JOBLOG_MAX
+        );
+    }
+
+    #[test]
+    fn overlap_log_defaults_follow_strategy_groups() {
+        assert_eq!(overlap_log_default(ZSTD_FAST), 6);
+        assert_eq!(overlap_log_default(ZSTD_DFAST), 6);
+        assert_eq!(overlap_log_default(ZSTD_GREEDY), 6);
+        assert_eq!(overlap_log_default(ZSTD_LAZY), 6);
+        assert_eq!(overlap_log_default(ZSTD_LAZY2), 7);
+        assert_eq!(overlap_log_default(ZSTD_BTLAZY2), 7);
+        assert_eq!(overlap_log_default(ZSTD_BTOPT), 8);
+        assert_eq!(overlap_log_default(ZSTD_BTULTRA), 8);
+        assert_eq!(overlap_log_default(ZSTD_BTULTRA2), 9);
+        assert_eq!(overlap_log_default(0), 6);
+
+        assert_eq!(overlap_log(0, ZSTD_BTULTRA2), 9);
+        assert_eq!(overlap_log(0, ZSTD_FAST), 6);
+        assert_eq!(overlap_log(5, ZSTD_BTULTRA2), 5);
+        assert_eq!(overlap_log(9, ZSTD_FAST), 9);
+    }
+
+    #[test]
+    fn overlap_size_uses_window_or_target_job_log_as_expected() {
+        assert_eq!(
+            compute_overlap_size(20, 25, ZSTD_FAST, 0, ZSTD_PS_DISABLE),
+            1usize << 17
+        );
+        assert_eq!(
+            compute_overlap_size(20, 25, ZSTD_BTULTRA2, 0, ZSTD_PS_DISABLE),
+            1usize << 20
+        );
+        assert_eq!(
+            compute_overlap_size(20, 25, ZSTD_FAST, 1, ZSTD_PS_DISABLE),
+            0
+        );
+        assert_eq!(
+            compute_overlap_size(30, 25, ZSTD_FAST, 0, ZSTD_PS_ENABLE),
+            1usize << 23
+        );
+        assert_eq!(
+            compute_overlap_size(30, 25, ZSTD_BTULTRA2, 0, ZSTD_PS_ENABLE),
+            1usize << 25
+        );
+    }
 }