feat(compress): move sequence-store branch policy into Rust

Move buildSeqStore's preloaded external-sequence, LDM, external-producer,
fallback, and ordinary matchfinder branch ordering into Rust. Keep the
private CCtx, matchfinder, LDM, and producer operations behind narrow C
callbacks, add ABI layout assertions, update the Rust boundary documentation,
and cover each policy branch with focused tests.

Test Plan:
- cargo fmt --manifest-path rust/Cargo.toml -- --check
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --lib (690 passed)
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings
- ulimit -v 41943040 && MAKEFLAGS=-j1 make -B -C programs -j1 zstd
- ulimit -v 41943040 && MAKEFLAGS=-j1 make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s (84 tests and both short fuzz rounds passed)

rust/src/zstd_compress.rs

@@ -177,14 +177,11 @@ type FrameChunkChecksumFn = unsafe extern "C" fn(*mut c_void, *const c_void, usi
 
 type BuildSeqStoreSkipFn = unsafe extern "C" fn(*mut c_void, usize);
 type BuildSeqStorePrepareFn = unsafe extern "C" fn(*mut c_void, *const c_void, usize);
-type BuildSeqStoreSelectFn = unsafe extern "C" fn(
-    *mut c_void,
-    *mut SeqStore_t,
-    *mut u32,
-    *const c_void,
-    usize,
-    *mut c_int,
-) -> usize;
+type BuildSeqStoreCompressFn =
+    unsafe extern "C" fn(*mut c_void, *mut SeqStore_t, *mut u32, *const c_void, usize) -> usize;
+type BuildSeqStoreTryExternalProducerFn =
+    unsafe extern "C" fn(*mut c_void, *const c_void, usize, *mut c_int, *mut c_int) -> usize;
+type BuildSeqStoreClearLdmFn = unsafe extern "C" fn(*mut c_void);
 
 /// Explicit projection for the sequence-store builder.
 ///
@@ -201,7 +198,15 @@ pub struct ZSTD_rust_buildSeqStoreState {
     validate_seq_store: c_int,
     skip_small_block: BuildSeqStoreSkipFn,
     prepare_match_state: BuildSeqStorePrepareFn,
-    select_sequences: BuildSeqStoreSelectFn,
+    has_external_sequences: c_int,
+    ldm_enabled: c_int,
+    has_external_sequence_producer: c_int,
+    enable_match_finder_fallback: c_int,
+    compress_external_sequences: BuildSeqStoreCompressFn,
+    compress_ldm: BuildSeqStoreCompressFn,
+    try_external_sequence_producer: BuildSeqStoreTryExternalProducerFn,
+    compress_internal: BuildSeqStoreCompressFn,
+    clear_ldm_seq_store: BuildSeqStoreClearLdmFn,
 }
 
 const _: () = {
@@ -223,12 +228,44 @@ const _: () = {
             == 5 * size_of::<usize>() + size_of::<c_uint>() + size_of::<c_int>()
     );
     assert!(
-        offset_of!(ZSTD_rust_buildSeqStoreState, select_sequences)
+        offset_of!(ZSTD_rust_buildSeqStoreState, has_external_sequences)
             == 6 * size_of::<usize>() + size_of::<c_uint>() + size_of::<c_int>()
     );
+    assert!(
+        offset_of!(ZSTD_rust_buildSeqStoreState, ldm_enabled)
+            == 6 * size_of::<usize>() + size_of::<c_uint>() + 2 * size_of::<c_int>()
+    );
+    assert!(
+        offset_of!(ZSTD_rust_buildSeqStoreState, has_external_sequence_producer)
+            == 6 * size_of::<usize>() + size_of::<c_uint>() + 3 * size_of::<c_int>()
+    );
+    assert!(
+        offset_of!(ZSTD_rust_buildSeqStoreState, enable_match_finder_fallback)
+            == 6 * size_of::<usize>() + size_of::<c_uint>() + 4 * size_of::<c_int>()
+    );
+    assert!(
+        offset_of!(ZSTD_rust_buildSeqStoreState, compress_external_sequences)
+            == 6 * size_of::<usize>() + size_of::<c_uint>() + 5 * size_of::<c_int>()
+    );
+    assert!(
+        offset_of!(ZSTD_rust_buildSeqStoreState, compress_ldm)
+            == 7 * size_of::<usize>() + size_of::<c_uint>() + 5 * size_of::<c_int>()
+    );
+    assert!(
+        offset_of!(ZSTD_rust_buildSeqStoreState, try_external_sequence_producer)
+            == size_of::<[usize; 8]>() + size_of::<c_uint>() + 5 * size_of::<c_int>()
+    );
+    assert!(
+        offset_of!(ZSTD_rust_buildSeqStoreState, compress_internal)
+            == 9 * size_of::<usize>() + size_of::<c_uint>() + 5 * size_of::<c_int>()
+    );
+    assert!(
+        offset_of!(ZSTD_rust_buildSeqStoreState, clear_ldm_seq_store)
+            == 10 * size_of::<usize>() + size_of::<c_uint>() + 5 * size_of::<c_int>()
+    );
     assert!(
         size_of::<ZSTD_rust_buildSeqStoreState>()
-            == 7 * size_of::<usize>() + size_of::<c_uint>() + size_of::<c_int>()
+            == 11 * size_of::<usize>() + size_of::<c_uint>() + 5 * size_of::<c_int>()
     );
 };
 
@@ -3567,7 +3604,84 @@ impl SingleBlockSeams {
     }
 }
 
-/// Rust-owned sequence-store boundary.  C callbacks perform the operations
+/// Rust-owned sequence-store branch policy. C callbacks perform the
+/// operations which need the private matchfinder or CCtx
+/// parameter/function-pointer state.
+#[allow(clippy::too_many_arguments)]
+unsafe fn build_seq_store_select_sequences_body_with(
+    state: &ZSTD_rust_buildSeqStoreState,
+    seq_store: *mut SeqStore_t,
+    next_rep: *mut u32,
+    src: *const c_void,
+    src_size: usize,
+    seq_store_complete: &mut c_int,
+) -> usize {
+    if seq_store.is_null()
+        || next_rep.is_null()
+        || src.is_null()
+        || state.compress_external_sequences as usize == 0
+        || state.compress_ldm as usize == 0
+        || state.try_external_sequence_producer as usize == 0
+        || state.compress_internal as usize == 0
+        || state.clear_ldm_seq_store as usize == 0
+    {
+        return ERROR(ZstdErrorCode::Generic);
+    }
+
+    *seq_store_complete = 0;
+    if state.has_external_sequences != 0 {
+        if state.has_external_sequence_producer != 0 {
+            return ERROR(ZstdErrorCode::ParameterCombinationUnsupported);
+        }
+        return unsafe {
+            (state.compress_external_sequences)(
+                state.callback_context,
+                seq_store,
+                next_rep,
+                src,
+                src_size,
+            )
+        };
+    }
+
+    if state.ldm_enabled != 0 {
+        if state.has_external_sequence_producer != 0 {
+            return ERROR(ZstdErrorCode::ParameterCombinationUnsupported);
+        }
+        return unsafe {
+            (state.compress_ldm)(state.callback_context, seq_store, next_rep, src, src_size)
+        };
+    }
+
+    if state.has_external_sequence_producer != 0 {
+        let mut allow_fallback = 0;
+        let producer_result = unsafe {
+            (state.try_external_sequence_producer)(
+                state.callback_context,
+                src,
+                src_size,
+                seq_store_complete,
+                &mut allow_fallback,
+            )
+        };
+        if *seq_store_complete != 0 {
+            unsafe { (state.clear_ldm_seq_store)(state.callback_context) };
+            return producer_result;
+        }
+        if allow_fallback == 0 || state.enable_match_finder_fallback == 0 {
+            return producer_result;
+        }
+        unsafe { (state.clear_ldm_seq_store)(state.callback_context) };
+        return unsafe {
+            (state.compress_internal)(state.callback_context, seq_store, next_rep, src, src_size)
+        };
+    }
+
+    unsafe { (state.clear_ldm_seq_store)(state.callback_context) };
+    unsafe { (state.compress_internal)(state.callback_context, seq_store, next_rep, src, src_size) }
+}
+
+/// Rust-owned sequence-store boundary. C callbacks perform the operations
 /// which need the private matchfinder or CCtx parameter/function-pointer state.
 #[allow(clippy::too_many_arguments)]
 unsafe fn build_seq_store_body_with(
@@ -3578,7 +3692,6 @@ unsafe fn build_seq_store_body_with(
     if state.seq_store.is_null()
         || state.skip_small_block as usize == 0
         || state.prepare_match_state as usize == 0
-        || state.select_sequences as usize == 0
     {
         return ERROR(ZstdErrorCode::Generic);
     }
@@ -3617,8 +3730,8 @@ unsafe fn build_seq_store_body_with(
 
     let mut seq_store_complete = 0;
     let last_literals_size = unsafe {
-        (state.select_sequences)(
-            state.callback_context,
+        build_seq_store_select_sequences_body_with(
+            state,
             state.seq_store,
             (*next_c_block).rep.as_mut_ptr(),
             src,
@@ -7700,6 +7813,259 @@ mod tests {
     const ZSTD_BTOPT: c_int = 7;
     const ZSTD_BTULTRA2: c_int = 9;
 
+    #[derive(Default)]
+    struct BuildSeqStoreSelectProbe {
+        events: Vec<&'static str>,
+        external_result: usize,
+        ldm_result: usize,
+        producer_result: usize,
+        producer_complete: c_int,
+        producer_allow_fallback: c_int,
+        internal_result: usize,
+    }
+
+    unsafe fn build_seq_store_select_probe(
+        context: *mut c_void,
+    ) -> &'static mut BuildSeqStoreSelectProbe {
+        unsafe { &mut *context.cast::<BuildSeqStoreSelectProbe>() }
+    }
+
+    unsafe extern "C" fn build_seq_store_test_skip(_context: *mut c_void, _src_size: usize) {}
+
+    unsafe extern "C" fn build_seq_store_test_prepare(
+        _context: *mut c_void,
+        _src: *const c_void,
+        _src_size: usize,
+    ) {
+    }
+
+    unsafe extern "C" fn build_seq_store_test_external(
+        context: *mut c_void,
+        _seq_store: *mut SeqStore_t,
+        _next_rep: *mut u32,
+        _src: *const c_void,
+        _src_size: usize,
+    ) -> usize {
+        let probe = unsafe { build_seq_store_select_probe(context) };
+        probe.events.push("external");
+        probe.external_result
+    }
+
+    unsafe extern "C" fn build_seq_store_test_ldm(
+        context: *mut c_void,
+        _seq_store: *mut SeqStore_t,
+        _next_rep: *mut u32,
+        _src: *const c_void,
+        _src_size: usize,
+    ) -> usize {
+        let probe = unsafe { build_seq_store_select_probe(context) };
+        probe.events.push("ldm");
+        probe.ldm_result
+    }
+
+    unsafe extern "C" fn build_seq_store_test_producer(
+        context: *mut c_void,
+        _src: *const c_void,
+        _src_size: usize,
+        seq_store_complete: *mut c_int,
+        allow_fallback: *mut c_int,
+    ) -> usize {
+        let probe = unsafe { build_seq_store_select_probe(context) };
+        probe.events.push("producer");
+        unsafe {
+            *seq_store_complete = probe.producer_complete;
+            *allow_fallback = probe.producer_allow_fallback;
+        }
+        probe.producer_result
+    }
+
+    unsafe extern "C" fn build_seq_store_test_internal(
+        context: *mut c_void,
+        _seq_store: *mut SeqStore_t,
+        _next_rep: *mut u32,
+        _src: *const c_void,
+        _src_size: usize,
+    ) -> usize {
+        let probe = unsafe { build_seq_store_select_probe(context) };
+        probe.events.push("internal");
+        probe.internal_result
+    }
+
+    unsafe extern "C" fn build_seq_store_test_clear(context: *mut c_void) {
+        let probe = unsafe { build_seq_store_select_probe(context) };
+        probe.events.push("clear");
+    }
+
+    fn build_seq_store_select_test_state(
+        probe: &mut BuildSeqStoreSelectProbe,
+        has_external_sequences: c_int,
+        ldm_enabled: c_int,
+        has_external_sequence_producer: c_int,
+        enable_match_finder_fallback: c_int,
+    ) -> ZSTD_rust_buildSeqStoreState {
+        ZSTD_rust_buildSeqStoreState {
+            seq_store: ptr::null_mut(),
+            prev_c_block: ptr::null_mut(),
+            next_c_block: ptr::null_mut(),
+            callback_context: (probe as *mut BuildSeqStoreSelectProbe).cast(),
+            min_match: 3,
+            validate_seq_store: 0,
+            skip_small_block: build_seq_store_test_skip,
+            prepare_match_state: build_seq_store_test_prepare,
+            has_external_sequences,
+            ldm_enabled,
+            has_external_sequence_producer,
+            enable_match_finder_fallback,
+            compress_external_sequences: build_seq_store_test_external,
+            compress_ldm: build_seq_store_test_ldm,
+            try_external_sequence_producer: build_seq_store_test_producer,
+            compress_internal: build_seq_store_test_internal,
+            clear_ldm_seq_store: build_seq_store_test_clear,
+        }
+    }
+
+    fn run_build_seq_store_select_test(state: &ZSTD_rust_buildSeqStoreState) -> (usize, c_int) {
+        let mut seq_store = unsafe { MaybeUninit::<SeqStore_t>::zeroed().assume_init() };
+        let mut next_rep = [0u32; ZSTD_REP_NUM];
+        let source = [0u8; 1];
+        let mut seq_store_complete = 99;
+        let result = unsafe {
+            build_seq_store_select_sequences_body_with(
+                state,
+                &mut seq_store,
+                next_rep.as_mut_ptr(),
+                source.as_ptr().cast(),
+                source.len(),
+                &mut seq_store_complete,
+            )
+        };
+        (result, seq_store_complete)
+    }
+
+    #[test]
+    fn build_seq_store_selects_preloaded_external_sequences_first() {
+        let mut probe = BuildSeqStoreSelectProbe {
+            external_result: 11,
+            ldm_result: 12,
+            internal_result: 13,
+            ..Default::default()
+        };
+        let state = build_seq_store_select_test_state(&mut probe, 1, 1, 0, 0);
+
+        let (result, seq_store_complete) = run_build_seq_store_select_test(&state);
+
+        assert_eq!(result, probe.external_result);
+        assert_eq!(seq_store_complete, 0);
+        assert_eq!(probe.events, ["external"]);
+    }
+
+    #[test]
+    fn build_seq_store_uses_ldm_when_no_external_sequences_are_pending() {
+        let mut probe = BuildSeqStoreSelectProbe {
+            ldm_result: 19,
+            internal_result: 23,
+            ..Default::default()
+        };
+        let state = build_seq_store_select_test_state(&mut probe, 0, 1, 0, 0);
+
+        let (result, seq_store_complete) = run_build_seq_store_select_test(&state);
+
+        assert_eq!(result, probe.ldm_result);
+        assert_eq!(seq_store_complete, 0);
+        assert_eq!(probe.events, ["ldm"]);
+    }
+
+    #[test]
+    fn build_seq_store_rejects_external_producer_with_preloaded_sequences_or_ldm() {
+        for (has_external_sequences, ldm_enabled) in [(1, 0), (0, 1)] {
+            let mut probe = BuildSeqStoreSelectProbe::default();
+            let state = build_seq_store_select_test_state(
+                &mut probe,
+                has_external_sequences,
+                ldm_enabled,
+                1,
+                1,
+            );
+
+            let (result, seq_store_complete) = run_build_seq_store_select_test(&state);
+
+            assert_eq!(
+                result,
+                ERROR(ZstdErrorCode::ParameterCombinationUnsupported)
+            );
+            assert_eq!(seq_store_complete, 0);
+            assert!(probe.events.is_empty());
+        }
+    }
+
+    #[test]
+    fn build_seq_store_clears_ldm_after_external_producer_completion() {
+        let mut probe = BuildSeqStoreSelectProbe {
+            producer_complete: 1,
+            producer_allow_fallback: 1,
+            producer_result: 0,
+            ..Default::default()
+        };
+        let state = build_seq_store_select_test_state(&mut probe, 0, 0, 1, 1);
+
+        let (result, seq_store_complete) = run_build_seq_store_select_test(&state);
+
+        assert_eq!(result, 0);
+        assert_eq!(seq_store_complete, 1);
+        assert_eq!(probe.events, ["producer", "clear"]);
+    }
+
+    #[test]
+    fn build_seq_store_does_not_fallback_when_producer_disallows_it() {
+        let producer_result = ERROR(ZstdErrorCode::SequenceProducerFailed);
+        let mut probe = BuildSeqStoreSelectProbe {
+            producer_result,
+            producer_allow_fallback: 1,
+            internal_result: 17,
+            ..Default::default()
+        };
+        let state = build_seq_store_select_test_state(&mut probe, 0, 0, 1, 0);
+
+        let (result, seq_store_complete) = run_build_seq_store_select_test(&state);
+
+        assert_eq!(result, producer_result);
+        assert_eq!(seq_store_complete, 0);
+        assert_eq!(probe.events, ["producer"]);
+    }
+
+    #[test]
+    fn build_seq_store_falls_back_after_external_producer_error() {
+        let producer_result = ERROR(ZstdErrorCode::SequenceProducerFailed);
+        let mut probe = BuildSeqStoreSelectProbe {
+            producer_result,
+            producer_allow_fallback: 1,
+            internal_result: 23,
+            ..Default::default()
+        };
+        let state = build_seq_store_select_test_state(&mut probe, 0, 0, 1, 1);
+
+        let (result, seq_store_complete) = run_build_seq_store_select_test(&state);
+
+        assert_eq!(result, probe.internal_result);
+        assert_eq!(seq_store_complete, 0);
+        assert_eq!(probe.events, ["producer", "clear", "internal"]);
+    }
+
+    #[test]
+    fn build_seq_store_clears_ldm_before_internal_compression() {
+        let mut probe = BuildSeqStoreSelectProbe {
+            internal_result: 29,
+            ..Default::default()
+        };
+        let state = build_seq_store_select_test_state(&mut probe, 0, 0, 0, 0);
+
+        let (result, seq_store_complete) = run_build_seq_store_select_test(&state);
+
+        assert_eq!(result, probe.internal_result);
+        assert_eq!(seq_store_complete, 0);
+        assert_eq!(probe.events, ["clear", "internal"]);
+    }
+
     #[derive(Default)]
     struct ResetMatchStateTestContext {
         events: Vec<c_int>,