fix(build): restore complete mixed Rust build matrix
`make all` exercises substantially more than the default zstd binary. It also builds compression-only and decompression-only archives, older contrib tools that compile program C shims directly, and the single-file amalgamations. The Rust migration had changed symbol ownership without updating every one of those feature and link boundaries. The first failure came from `fileio_asyncio`: it is always compiled, but its codec bindings unconditionally referenced both `zstd_compress` and `zstd_decompress`. A compression-only archive therefore required disabled decoder modules, and the decompression-only configuration had the symmetric problem. Gate the concrete codec imports, callback types, helpers, and exports with their Cargo features. Keep the format probe compilable without the legacy decoder predicate when decompression is disabled. Once that boundary compiled, the remaining `make all` paths exposed related integration gaps. Move the C decompression projection declarations out of the compression preprocessor block, while leaving destination callback types shared. Link the Rust CLI helpers archive into zlibWrapper, pzstd, and largeNbDicts, whose util/time/data-generator C files are now declaration shims rather than implementations. The generated single-file C sources also call Rust-owned symbols now. Build and link an appropriately featured Rust archive in their native smoke tests, and include the pool configuration shim in the decoder case. The full amalgamation combines `zstd_lazy.c` and `zstd_opt.c` into one translation unit, so guard their otherwise translation-unit-local dictionary mode enum against duplicate definition. A Rust-backed amalgamation is no longer a standalone Emscripten input. Remove the obsolete emcc/Docker path and report that limitation explicitly; restoring the WebAssembly smoke test requires a Rust WebAssembly archive and a defined cross-language amalgamation contract. Test Plan: - `cargo check --manifest-path rust/Cargo.toml --no-default-features --features compression` -- passed - `cargo check --manifest-path rust/Cargo.toml --no-default-features --features decompression` -- passed - `sh -n build/single_file_libs/build_decoder_test.sh build/single_file_libs/build_library_test.sh` -- passed - `make all` -- passed, including native single-file and seekable-format tests - `git diff --cached --check` -- passed
This commit is contained in:
@@ -27,9 +27,10 @@ use std::sync::{Arc, Condvar, Mutex};
|
||||
use std::thread::{self, JoinHandle};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
#[cfg(feature = "compression")]
|
||||
use crate::zstd_compress::{ZSTD_inBuffer, ZSTD_outBuffer};
|
||||
|
||||
#[cfg(not(test))]
|
||||
#[cfg(all(feature = "compression", not(test)))]
|
||||
unsafe extern "C" {
|
||||
fn ZSTD_toFlushNow(cctx: *mut c_void) -> usize;
|
||||
}
|
||||
@@ -39,6 +40,7 @@ const IO_QUEUE_SIZE: usize = MAX_IO_JOBS - 2;
|
||||
const SPARSE_SEGMENT_SIZE: usize = 32 * 1024;
|
||||
const SPARSE_SKIP_CHUNK: u64 = 1 << 30;
|
||||
const PASS_THROUGH_MAX_BLOCK_SIZE: usize = 64 * 1024;
|
||||
#[cfg(feature = "decompression")]
|
||||
const ZSTD_FRAMEHEADERSIZE_MAX: usize = 18;
|
||||
|
||||
pub const FIO_RUST_ZSTD_FRAME_OK: c_int = 0;
|
||||
@@ -126,13 +128,17 @@ type FIO_rust_zstd_frame_premature_end_fn = unsafe extern "C" fn(*mut c_void, *c
|
||||
pub type FIO_rust_pass_through_fn = unsafe extern "C" fn(*mut c_void) -> c_int;
|
||||
pub type FIO_rust_decompress_status_fn = unsafe extern "C" fn(*mut c_void, c_int, *const c_char);
|
||||
pub type FIO_rust_decompress_finish_fn = unsafe extern "C" fn(*mut c_void, *const c_char, u64);
|
||||
#[cfg(feature = "decompression")]
|
||||
type FIO_zstd_reset_fn = unsafe extern "C" fn(*mut c_void, c_int) -> usize;
|
||||
#[cfg(feature = "decompression")]
|
||||
type FIO_zstd_decompress_fn = unsafe extern "C" fn(
|
||||
*mut c_void,
|
||||
*mut crate::zstd_decompress::ZSTD_outBuffer,
|
||||
*mut crate::zstd_decompress::ZSTD_inBuffer,
|
||||
) -> usize;
|
||||
#[cfg(feature = "decompression")]
|
||||
type FIO_zstd_in_size_fn = extern "C" fn() -> usize;
|
||||
#[cfg(feature = "decompression")]
|
||||
type FIO_zstd_is_frame_fn = unsafe extern "C" fn(*const c_void, usize) -> c_uint;
|
||||
|
||||
/// Rust owns the default zstd-frame result policy. C supplies only the exact
|
||||
@@ -4031,13 +4037,16 @@ unsafe fn zstd_adaptive_iteration(
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "compression")]
|
||||
type FIO_rust_zstd_to_flush_now_fn = unsafe extern "C" fn(*mut c_void) -> usize;
|
||||
#[cfg(feature = "compression")]
|
||||
type FIO_rust_zstd_compress_stream2_fn =
|
||||
unsafe extern "C" fn(*mut c_void, *mut ZSTD_outBuffer, *mut ZSTD_inBuffer, c_int) -> usize;
|
||||
|
||||
/// Build the public stream-buffer views, call the Rust compressor, and publish
|
||||
/// the same scalar results as the former C-only callback. The codec context
|
||||
/// stays opaque; only the public buffer ABI crosses into the compressor.
|
||||
#[cfg(feature = "compression")]
|
||||
unsafe fn fio_zstd_compress_stream_with(
|
||||
cctx: *mut c_void,
|
||||
directive: c_int,
|
||||
@@ -4085,7 +4094,7 @@ unsafe fn fio_zstd_compress_stream_with(
|
||||
/// the iteration callback's private adaptive/diagnostic context; this callback
|
||||
/// receives the opaque `ZSTD_CCtx` directly and delegates to Rust's public
|
||||
/// `ZSTD_compressStream2` implementation.
|
||||
#[cfg(not(test))]
|
||||
#[cfg(all(feature = "compression", not(test)))]
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn FIO_rust_zstd_compressStream(
|
||||
cctx: *mut c_void,
|
||||
@@ -4935,6 +4944,7 @@ pub unsafe extern "C" fn FIO_rust_compressLzmaFrame(
|
||||
/// parameters so the C-owned `dRess_t` never crosses into Rust. The input
|
||||
/// pool is advanced only after a successful decoder call; in particular, an
|
||||
/// error leaves the current input buffer untouched for `FIO_zstdErrorHelp()`.
|
||||
#[cfg(feature = "decompression")]
|
||||
unsafe fn decompress_zstd_frame_with(
|
||||
f_ctx: *mut c_void,
|
||||
dctx: *mut c_void,
|
||||
@@ -5030,6 +5040,7 @@ unsafe fn decompress_zstd_frame_with(
|
||||
/// leaves a non-zstd header or a short trailing buffer untouched for C's
|
||||
/// mixed-format dispatcher. Completed-frame output remains accounted for if
|
||||
/// a later frame reports an error.
|
||||
#[cfg(feature = "decompression")]
|
||||
unsafe fn decompress_zstd_frames_with(
|
||||
f_ctx: *mut c_void,
|
||||
dctx: *mut c_void,
|
||||
@@ -5098,6 +5109,7 @@ unsafe fn decompress_zstd_frames_with(
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
unsafe extern "C" fn fio_zstd_reset(dctx: *mut c_void, reset: c_int) -> usize {
|
||||
unsafe {
|
||||
crate::zstd_decompress::ZSTD_DCtx_reset(
|
||||
@@ -5107,6 +5119,7 @@ unsafe extern "C" fn fio_zstd_reset(dctx: *mut c_void, reset: c_int) -> usize {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
unsafe extern "C" fn fio_zstd_decompress(
|
||||
dctx: *mut c_void,
|
||||
output: *mut crate::zstd_decompress::ZSTD_outBuffer,
|
||||
@@ -5122,6 +5135,7 @@ unsafe extern "C" fn fio_zstd_decompress(
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
#[cfg(feature = "decompression")]
|
||||
pub unsafe extern "C" fn FIO_rust_decompressZstdFrame(
|
||||
f_ctx: *mut c_void,
|
||||
dctx: *mut c_void,
|
||||
@@ -5152,6 +5166,7 @@ pub unsafe extern "C" fn FIO_rust_decompressZstdFrame(
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
#[cfg(feature = "decompression")]
|
||||
pub unsafe extern "C" fn FIO_rust_decompressZstdFrames(
|
||||
f_ctx: *mut c_void,
|
||||
dctx: *mut c_void,
|
||||
@@ -5182,9 +5197,11 @@ pub unsafe extern "C" fn FIO_rust_decompressZstdFrames(
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
const FIO_ERROR_FRAME_DECODING: u64 = u64::MAX - 1;
|
||||
|
||||
#[inline]
|
||||
#[cfg(feature = "decompression")]
|
||||
unsafe fn zstd_frame_policy_result(
|
||||
callback_context: *mut c_void,
|
||||
src_file_name: *const c_char,
|
||||
@@ -5211,6 +5228,7 @@ unsafe fn zstd_frame_policy_result(
|
||||
/// Run the default zstd-frame loop and apply its historical result/error
|
||||
/// policy. C retains only exact diagnostics and private resource callbacks.
|
||||
#[no_mangle]
|
||||
#[cfg(feature = "decompression")]
|
||||
pub unsafe extern "C" fn FIO_rust_decompressZstdFramePolicy(
|
||||
state: *const FIO_rust_zstd_frame_policy_state,
|
||||
) -> u64 {
|
||||
@@ -5797,14 +5815,14 @@ where
|
||||
|
||||
#[inline]
|
||||
unsafe fn is_zstd_frame_for_dispatch(buffer: &[u8]) -> bool {
|
||||
#[cfg(test)]
|
||||
#[cfg(any(test, not(feature = "decompression")))]
|
||||
{
|
||||
/* Standalone Rust tests do not link the C legacy-decoder shim. The
|
||||
* dispatch tests only need the modern frame magic; production keeps
|
||||
* the complete public predicate below. */
|
||||
buffer.starts_with(&[0x28, 0xB5, 0x2F, 0xFD])
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
#[cfg(all(feature = "decompression", not(test)))]
|
||||
{
|
||||
unsafe { crate::zstd_decompress::ZSTD_isFrame(buffer.as_ptr().cast(), buffer.len()) != 0 }
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user