fix(build): restore complete mixed Rust build matrix
`make all` exercises substantially more than the default zstd binary. It also builds compression-only and decompression-only archives, older contrib tools that compile program C shims directly, and the single-file amalgamations. The Rust migration had changed symbol ownership without updating every one of those feature and link boundaries. The first failure came from `fileio_asyncio`: it is always compiled, but its codec bindings unconditionally referenced both `zstd_compress` and `zstd_decompress`. A compression-only archive therefore required disabled decoder modules, and the decompression-only configuration had the symmetric problem. Gate the concrete codec imports, callback types, helpers, and exports with their Cargo features. Keep the format probe compilable without the legacy decoder predicate when decompression is disabled. Once that boundary compiled, the remaining `make all` paths exposed related integration gaps. Move the C decompression projection declarations out of the compression preprocessor block, while leaving destination callback types shared. Link the Rust CLI helpers archive into zlibWrapper, pzstd, and largeNbDicts, whose util/time/data-generator C files are now declaration shims rather than implementations. The generated single-file C sources also call Rust-owned symbols now. Build and link an appropriately featured Rust archive in their native smoke tests, and include the pool configuration shim in the decoder case. The full amalgamation combines `zstd_lazy.c` and `zstd_opt.c` into one translation unit, so guard their otherwise translation-unit-local dictionary mode enum against duplicate definition. A Rust-backed amalgamation is no longer a standalone Emscripten input. Remove the obsolete emcc/Docker path and report that limitation explicitly; restoring the WebAssembly smoke test requires a Rust WebAssembly archive and a defined cross-language amalgamation contract. Test Plan: - `cargo check --manifest-path rust/Cargo.toml --no-default-features --features compression` -- passed - `cargo check --manifest-path rust/Cargo.toml --no-default-features --features decompression` -- passed - `sh -n build/single_file_libs/build_decoder_test.sh build/single_file_libs/build_library_test.sh` -- passed - `make all` -- passed, including native single-file and seekable-format tests - `git diff --cached --check` -- passed
This commit is contained in:
@@ -3,58 +3,8 @@
|
||||
# Temporary compiled binary
|
||||
OUT_FILE="tempbin"
|
||||
|
||||
# Optional temporary compiled WebAssembly
|
||||
OUT_WASM="temp.wasm"
|
||||
|
||||
# Source files to compile using Emscripten.
|
||||
IN_FILES="examples/emscripten.c"
|
||||
|
||||
# Emscripten build using emcc.
|
||||
emscripten_emcc_build() {
|
||||
# Compile the same example as above
|
||||
CC_FLAGS="-Wall -Wextra -Wshadow -Werror -Os -g0 -flto"
|
||||
emcc $CC_FLAGS -s WASM=1 -I. -o $OUT_WASM $IN_FILES
|
||||
# Did compilation work?
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Compiling ${IN_FILES}: FAILED"
|
||||
exit 1
|
||||
fi
|
||||
echo "Compiling ${IN_FILES}: PASSED"
|
||||
rm -f $OUT_WASM
|
||||
}
|
||||
|
||||
# Emscripten build using docker.
|
||||
emscripten_docker_build() {
|
||||
docker container run --rm \
|
||||
--volume $PWD:/code \
|
||||
--workdir /code \
|
||||
emscripten/emsdk:latest \
|
||||
emcc $CC_FLAGS -s WASM=1 -I. -o $OUT_WASM $IN_FILES
|
||||
# Did compilation work?
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Compiling ${IN_FILES} (using docker): FAILED"
|
||||
exit 1
|
||||
fi
|
||||
echo "Compiling ${IN_FILES} (using docker): PASSED"
|
||||
rm -f $OUT_WASM
|
||||
}
|
||||
|
||||
# Try Emscripten build using emcc or docker.
|
||||
try_emscripten_build() {
|
||||
which emcc > /dev/null
|
||||
if [ $? -eq 0 ]; then
|
||||
emscripten_emcc_build
|
||||
return $?
|
||||
fi
|
||||
|
||||
which docker > /dev/null
|
||||
if [ $? -eq 0 ]; then
|
||||
emscripten_docker_build
|
||||
return $?
|
||||
fi
|
||||
|
||||
echo "(Skipping Emscripten test)"
|
||||
}
|
||||
RUST_TARGET_DIR="../../rust/target/single-file-decompress"
|
||||
RUST_ARCHIVE="$RUST_TARGET_DIR/release/libzstd_rs.a"
|
||||
|
||||
# Amalgamate the sources
|
||||
./create_single_file_decoder.sh
|
||||
@@ -65,8 +15,17 @@ if [ $? -ne 0 ]; then
|
||||
fi
|
||||
echo "Single file decoder creation script: PASSED"
|
||||
|
||||
# Build the Rust decoder implementation referenced by the amalgamated C shims.
|
||||
${CARGO:-cargo} build --manifest-path ../../rust/Cargo.toml --release \
|
||||
--target-dir "$RUST_TARGET_DIR" --no-default-features --features decompression
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Building Rust decoder archive: FAILED"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Compile the generated output
|
||||
cc -Wall -Wextra -Wshadow -Werror -Os -g0 -o $OUT_FILE examples/simple.c
|
||||
cc -Wall -Wextra -Wshadow -Werror -Os -g0 -I../../lib -o $OUT_FILE \
|
||||
examples/simple.c ../../lib/common/pool.c "$RUST_ARCHIVE"
|
||||
# Did compilation work?
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Compiling simple.c: FAILED"
|
||||
@@ -85,7 +44,7 @@ if [ $retVal -ne 0 ]; then
|
||||
fi
|
||||
echo "Running simple.c: PASSED"
|
||||
|
||||
# Try Emscripten build if emcc or docker command is available.
|
||||
try_emscripten_build
|
||||
# The generated C now calls Rust, so it is no longer a standalone Emscripten input.
|
||||
echo "(Skipping Emscripten test for Rust-backed amalgamation)"
|
||||
|
||||
exit 0
|
||||
|
||||
@@ -6,58 +6,8 @@ ZSTD_SRC_ROOT="../../lib"
|
||||
# Temporary compiled binary
|
||||
OUT_FILE="tempbin"
|
||||
|
||||
# Optional temporary compiled WebAssembly
|
||||
OUT_WASM="temp.wasm"
|
||||
|
||||
# Source files to compile using Emscripten.
|
||||
IN_FILES="zstd.c examples/roundtrip.c"
|
||||
|
||||
# Emscripten build using emcc.
|
||||
emscripten_emcc_build() {
|
||||
# Compile the same example as above
|
||||
CC_FLAGS="-Wall -Wextra -Wshadow -Werror -Os -g0 -flto"
|
||||
emcc $CC_FLAGS -s WASM=1 -I. -o $OUT_WASM $IN_FILES
|
||||
# Did compilation work?
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Compiling ${IN_FILES}: FAILED"
|
||||
exit 1
|
||||
fi
|
||||
echo "Compiling ${IN_FILES}: PASSED"
|
||||
rm -f $OUT_WASM
|
||||
}
|
||||
|
||||
# Emscripten build using docker.
|
||||
emscripten_docker_build() {
|
||||
docker container run --rm \
|
||||
--volume $PWD:/code \
|
||||
--workdir /code \
|
||||
emscripten/emsdk:latest \
|
||||
emcc $CC_FLAGS -s WASM=1 -I. -o $OUT_WASM $IN_FILES
|
||||
# Did compilation work?
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Compiling ${IN_FILES} (using docker): FAILED"
|
||||
exit 1
|
||||
fi
|
||||
echo "Compiling ${IN_FILES} (using docker): PASSED"
|
||||
rm -f $OUT_WASM
|
||||
}
|
||||
|
||||
# Try Emscripten build using emcc or docker.
|
||||
try_emscripten_build() {
|
||||
which emcc > /dev/null
|
||||
if [ $? -eq 0 ]; then
|
||||
emscripten_emcc_build
|
||||
return $?
|
||||
fi
|
||||
|
||||
which docker > /dev/null
|
||||
if [ $? -eq 0 ]; then
|
||||
emscripten_docker_build
|
||||
return $?
|
||||
fi
|
||||
|
||||
echo "(Skipping Emscripten test)"
|
||||
}
|
||||
RUST_TARGET_DIR="../../rust/target/single-file-library"
|
||||
RUST_ARCHIVE="$RUST_TARGET_DIR/release/libzstd_rs.a"
|
||||
|
||||
# Amalgamate the sources
|
||||
./create_single_file_library.sh
|
||||
@@ -72,8 +22,18 @@ echo "Single file library creation script: PASSED"
|
||||
cp "$ZSTD_SRC_ROOT/zstd.h" examples/zstd.h
|
||||
cp "$ZSTD_SRC_ROOT/zstd_errors.h" examples/zstd_errors.h
|
||||
|
||||
# Build the Rust implementation referenced by the amalgamated C shims.
|
||||
${CARGO:-cargo} build --manifest-path ../../rust/Cargo.toml --release \
|
||||
--target-dir "$RUST_TARGET_DIR"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Building Rust library archive: FAILED"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Compile the generated output
|
||||
cc -Wall -Wextra -Werror -Wshadow -pthread -I. -Os -g0 -o $OUT_FILE zstd.c examples/roundtrip.c
|
||||
cc -Wall -Wextra -Werror -Wshadow \
|
||||
-Wno-unused-variable -Wno-unused-parameter -Wno-unused-but-set-variable \
|
||||
-pthread -I. -Os -g0 -o $OUT_FILE zstd.c examples/roundtrip.c "$RUST_ARCHIVE"
|
||||
# Did compilation work?
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Compiling roundtrip.c: FAILED"
|
||||
@@ -92,7 +52,7 @@ if [ $retVal -ne 0 ]; then
|
||||
fi
|
||||
echo "Running roundtrip.c: PASSED"
|
||||
|
||||
# Try Emscripten build if emcc or docker command is available.
|
||||
try_emscripten_build
|
||||
# The generated C now calls Rust, so it is no longer a standalone Emscripten input.
|
||||
echo "(Skipping Emscripten test for Rust-backed amalgamation)"
|
||||
|
||||
exit 0
|
||||
|
||||
@@ -9,6 +9,20 @@
|
||||
|
||||
PROGDIR = ../../programs
|
||||
LIBDIR = ../../lib
|
||||
CARGO ?= cargo
|
||||
RUSTDIR = ../../rust
|
||||
RUST_CLI_DIR = $(RUSTDIR)/cli
|
||||
RUST_CLI_MANIFEST = $(RUST_CLI_DIR)/Cargo.toml
|
||||
RUST_CLI_HELPER_SOURCES = $(RUST_CLI_MANIFEST) $(RUST_CLI_DIR)/Cargo.lock \
|
||||
$(RUST_CLI_DIR)/src/lib.rs \
|
||||
$(RUSTDIR)/src/timefn.rs $(RUSTDIR)/src/benchfn.rs \
|
||||
$(RUSTDIR)/src/datagen.rs $(RUSTDIR)/src/lorem.rs \
|
||||
$(RUSTDIR)/src/util.rs
|
||||
RUST_CLI_HELPERS_TARGET_DIR = $(RUSTDIR)/target/cli-helpers
|
||||
RUST_CLI_HELPERS_STATICLIB = $(RUST_CLI_HELPERS_TARGET_DIR)/release/libzstd_cli_rs.a
|
||||
RUST_CLI_HELPERS_CARGO_FLAGS = --manifest-path $(RUST_CLI_MANIFEST) --release \
|
||||
--target-dir $(RUST_CLI_HELPERS_TARGET_DIR) \
|
||||
--no-default-features --features helpers
|
||||
|
||||
LIBZSTD = $(LIBDIR)/libzstd.a
|
||||
|
||||
@@ -28,13 +42,16 @@ default: largeNbDicts
|
||||
|
||||
all : largeNbDicts
|
||||
|
||||
largeNbDicts: util.o timefn.o benchfn.o datagen.o xxhash.o largeNbDicts.c $(LIBZSTD)
|
||||
largeNbDicts: util.o timefn.o benchfn.o datagen.o xxhash.o largeNbDicts.c $(LIBZSTD) $(RUST_CLI_HELPERS_STATICLIB)
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
.PHONY: $(LIBZSTD)
|
||||
$(LIBZSTD):
|
||||
$(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)"
|
||||
|
||||
$(RUST_CLI_HELPERS_STATICLIB): $(RUST_CLI_HELPER_SOURCES)
|
||||
$(CARGO) build $(RUST_CLI_HELPERS_CARGO_FLAGS)
|
||||
|
||||
benchfn.o: $(PROGDIR)/benchfn.c
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
|
||||
|
||||
|
||||
+22
-3
@@ -14,6 +14,20 @@ BINDIR := $(PREFIX)/bin
|
||||
|
||||
ZSTDDIR = ../../lib
|
||||
PROGDIR = ../../programs
|
||||
CARGO ?= cargo
|
||||
RUSTDIR = ../../rust
|
||||
RUST_CLI_DIR = $(RUSTDIR)/cli
|
||||
RUST_CLI_MANIFEST = $(RUST_CLI_DIR)/Cargo.toml
|
||||
RUST_CLI_HELPER_SOURCES = $(RUST_CLI_MANIFEST) $(RUST_CLI_DIR)/Cargo.lock \
|
||||
$(RUST_CLI_DIR)/src/lib.rs \
|
||||
$(RUSTDIR)/src/timefn.rs $(RUSTDIR)/src/benchfn.rs \
|
||||
$(RUSTDIR)/src/datagen.rs $(RUSTDIR)/src/lorem.rs \
|
||||
$(RUSTDIR)/src/util.rs
|
||||
RUST_CLI_HELPERS_TARGET_DIR = $(RUSTDIR)/target/cli-helpers
|
||||
RUST_CLI_HELPERS_STATICLIB = $(RUST_CLI_HELPERS_TARGET_DIR)/release/libzstd_cli_rs.a
|
||||
RUST_CLI_HELPERS_CARGO_FLAGS = --manifest-path $(RUST_CLI_MANIFEST) --release \
|
||||
--target-dir $(RUST_CLI_HELPERS_TARGET_DIR) \
|
||||
--no-default-features --features helpers
|
||||
|
||||
# External program to use to run tests, e.g. qemu or valgrind
|
||||
TESTPROG ?=
|
||||
@@ -164,7 +178,7 @@ roundtripcheck: roundtrip check
|
||||
$(TESTPROG) ./test/RoundTripTest$(EXT) $(TESTFLAGS)
|
||||
|
||||
# Build the main binary
|
||||
pzstd$(EXT): main.o $(PROGDIR)/util.o Options.o Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a
|
||||
pzstd$(EXT): main.o $(PROGDIR)/util.o Options.o Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a $(RUST_CLI_HELPERS_STATICLIB)
|
||||
$(LD_COMMAND)
|
||||
|
||||
# Target that depends on all the tests
|
||||
@@ -183,17 +197,22 @@ roundtrip: test/RoundTripTest$(EXT)
|
||||
$(ZSTDDIR)/libzstd.a:
|
||||
CFLAGS="$(ALL_CFLAGS)" LDFLAGS="$(ALL_LDFLAGS)" $(MAKE) -C $(ZSTDDIR) libzstd.a
|
||||
|
||||
$(RUST_CLI_HELPERS_STATICLIB): $(RUST_CLI_HELPER_SOURCES)
|
||||
$(CARGO) build $(RUST_CLI_HELPERS_CARGO_FLAGS)
|
||||
|
||||
# Rules to build the tests
|
||||
test/RoundTripTest$(EXT): test/RoundTripTest.o $(PROGDIR)/datagen.o \
|
||||
$(PROGDIR)/util.o Options.o \
|
||||
Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a
|
||||
Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a \
|
||||
$(RUST_CLI_HELPERS_STATICLIB)
|
||||
$(LD_COMMAND)
|
||||
|
||||
test/%Test$(EXT): PZSTD_LDFLAGS += $(GTEST_LIB)
|
||||
test/%Test$(EXT): LIBS += -lgtest -lgtest_main
|
||||
test/%Test$(EXT): test/%Test.o $(PROGDIR)/datagen.o \
|
||||
$(PROGDIR)/util.o Options.o Pzstd.o \
|
||||
SkippableFrame.o $(ZSTDDIR)/libzstd.a
|
||||
SkippableFrame.o $(ZSTDDIR)/libzstd.a \
|
||||
$(RUST_CLI_HELPERS_STATICLIB)
|
||||
$(LD_COMMAND)
|
||||
|
||||
utils/test/%Test$(EXT): PZSTD_LDFLAGS += $(GTEST_LIB)
|
||||
|
||||
@@ -67,12 +67,15 @@ enum {
|
||||
ZSTD_rust_search_rowHash = 2,
|
||||
};
|
||||
|
||||
#ifndef ZSTD_RUST_DICT_MODE_ENUM
|
||||
#define ZSTD_RUST_DICT_MODE_ENUM
|
||||
enum {
|
||||
ZSTD_rust_dict_noDict = 0,
|
||||
ZSTD_rust_dict_extDict = 1,
|
||||
ZSTD_rust_dict_dictMatchState = 2,
|
||||
ZSTD_rust_dict_dedicatedDictSearch = 3,
|
||||
};
|
||||
#endif
|
||||
|
||||
static void ZSTD_rustLazyState_init(
|
||||
ZSTD_RustLazyState* const out,
|
||||
|
||||
@@ -145,11 +145,14 @@ size_t ZSTD_rust_compressBlock_btultra2(
|
||||
ZSTD_RustOptState* state, void* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
#ifndef ZSTD_RUST_DICT_MODE_ENUM
|
||||
#define ZSTD_RUST_DICT_MODE_ENUM
|
||||
enum {
|
||||
ZSTD_rust_dict_noDict = 0,
|
||||
ZSTD_rust_dict_extDict = 1,
|
||||
ZSTD_rust_dict_dictMatchState = 2,
|
||||
};
|
||||
#endif
|
||||
|
||||
void ZSTD_updateTree(ZSTD_MatchState_t* const ms, const BYTE* const ip,
|
||||
const BYTE* const iend)
|
||||
|
||||
+14
-5
@@ -1387,6 +1387,12 @@ static void FIO_displayMultipleFileSummary(const FIO_ctx_t* fCtx, int summaryKin
|
||||
summaryKind, &callbacks);
|
||||
}
|
||||
|
||||
typedef int (*FIO_rust_multiple_destination_warning_fn)(void* opaque);
|
||||
typedef void* (*FIO_rust_multiple_destination_open_fn)(void* opaque);
|
||||
typedef void (*FIO_rust_multiple_destination_attach_fn)(
|
||||
void* opaque, void* destination);
|
||||
typedef int (*FIO_rust_multiple_destination_close_fn)(void* opaque);
|
||||
|
||||
#ifndef ZSTD_NOCOMPRESS
|
||||
|
||||
/* **********************************************************************
|
||||
@@ -1478,11 +1484,6 @@ typedef char FIO_rust_compress_multiple_projection_size[
|
||||
int FIO_rust_compressMultipleFilenames(
|
||||
const FIO_rust_compress_multiple_projection_t* projection);
|
||||
|
||||
typedef int (*FIO_rust_multiple_destination_warning_fn)(void* opaque);
|
||||
typedef void* (*FIO_rust_multiple_destination_open_fn)(void* opaque);
|
||||
typedef void (*FIO_rust_multiple_destination_attach_fn)(
|
||||
void* opaque, void* destination);
|
||||
typedef int (*FIO_rust_multiple_destination_close_fn)(void* opaque);
|
||||
typedef struct {
|
||||
const FIO_rust_compress_multiple_projection_t* files;
|
||||
FIO_rust_multiple_destination_warning_fn warning;
|
||||
@@ -1555,6 +1556,10 @@ typedef char FIO_rust_compress_multiple_separate_projection_size[
|
||||
int FIO_rust_compressMultipleSeparateFilenames(
|
||||
const FIO_rust_compress_multiple_separate_projection_t* projection);
|
||||
|
||||
#endif /* #ifndef ZSTD_NOCOMPRESS */
|
||||
|
||||
#ifndef ZSTD_NODECOMPRESS
|
||||
|
||||
typedef int (*FIO_rust_decompress_multiple_file_fn)(
|
||||
void* opaque, const char* outFileName, const char* srcFileName);
|
||||
typedef struct {
|
||||
@@ -1738,6 +1743,10 @@ typedef char FIO_rust_decompress_file_projection_size[
|
||||
int FIO_rust_decompressFilename(
|
||||
const FIO_rust_decompress_file_projection_t* projection);
|
||||
|
||||
#endif /* #ifndef ZSTD_NODECOMPRESS */
|
||||
|
||||
#ifndef ZSTD_NOCOMPRESS
|
||||
|
||||
enum {
|
||||
FIO_RUST_ZSTD_OK = 0,
|
||||
FIO_RUST_ZSTD_COMPRESS_ERROR = 1,
|
||||
|
||||
@@ -27,9 +27,10 @@ use std::sync::{Arc, Condvar, Mutex};
|
||||
use std::thread::{self, JoinHandle};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
#[cfg(feature = "compression")]
|
||||
use crate::zstd_compress::{ZSTD_inBuffer, ZSTD_outBuffer};
|
||||
|
||||
#[cfg(not(test))]
|
||||
#[cfg(all(feature = "compression", not(test)))]
|
||||
unsafe extern "C" {
|
||||
fn ZSTD_toFlushNow(cctx: *mut c_void) -> usize;
|
||||
}
|
||||
@@ -39,6 +40,7 @@ const IO_QUEUE_SIZE: usize = MAX_IO_JOBS - 2;
|
||||
const SPARSE_SEGMENT_SIZE: usize = 32 * 1024;
|
||||
const SPARSE_SKIP_CHUNK: u64 = 1 << 30;
|
||||
const PASS_THROUGH_MAX_BLOCK_SIZE: usize = 64 * 1024;
|
||||
#[cfg(feature = "decompression")]
|
||||
const ZSTD_FRAMEHEADERSIZE_MAX: usize = 18;
|
||||
|
||||
pub const FIO_RUST_ZSTD_FRAME_OK: c_int = 0;
|
||||
@@ -126,13 +128,17 @@ type FIO_rust_zstd_frame_premature_end_fn = unsafe extern "C" fn(*mut c_void, *c
|
||||
pub type FIO_rust_pass_through_fn = unsafe extern "C" fn(*mut c_void) -> c_int;
|
||||
pub type FIO_rust_decompress_status_fn = unsafe extern "C" fn(*mut c_void, c_int, *const c_char);
|
||||
pub type FIO_rust_decompress_finish_fn = unsafe extern "C" fn(*mut c_void, *const c_char, u64);
|
||||
#[cfg(feature = "decompression")]
|
||||
type FIO_zstd_reset_fn = unsafe extern "C" fn(*mut c_void, c_int) -> usize;
|
||||
#[cfg(feature = "decompression")]
|
||||
type FIO_zstd_decompress_fn = unsafe extern "C" fn(
|
||||
*mut c_void,
|
||||
*mut crate::zstd_decompress::ZSTD_outBuffer,
|
||||
*mut crate::zstd_decompress::ZSTD_inBuffer,
|
||||
) -> usize;
|
||||
#[cfg(feature = "decompression")]
|
||||
type FIO_zstd_in_size_fn = extern "C" fn() -> usize;
|
||||
#[cfg(feature = "decompression")]
|
||||
type FIO_zstd_is_frame_fn = unsafe extern "C" fn(*const c_void, usize) -> c_uint;
|
||||
|
||||
/// Rust owns the default zstd-frame result policy. C supplies only the exact
|
||||
@@ -4031,13 +4037,16 @@ unsafe fn zstd_adaptive_iteration(
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "compression")]
|
||||
type FIO_rust_zstd_to_flush_now_fn = unsafe extern "C" fn(*mut c_void) -> usize;
|
||||
#[cfg(feature = "compression")]
|
||||
type FIO_rust_zstd_compress_stream2_fn =
|
||||
unsafe extern "C" fn(*mut c_void, *mut ZSTD_outBuffer, *mut ZSTD_inBuffer, c_int) -> usize;
|
||||
|
||||
/// Build the public stream-buffer views, call the Rust compressor, and publish
|
||||
/// the same scalar results as the former C-only callback. The codec context
|
||||
/// stays opaque; only the public buffer ABI crosses into the compressor.
|
||||
#[cfg(feature = "compression")]
|
||||
unsafe fn fio_zstd_compress_stream_with(
|
||||
cctx: *mut c_void,
|
||||
directive: c_int,
|
||||
@@ -4085,7 +4094,7 @@ unsafe fn fio_zstd_compress_stream_with(
|
||||
/// the iteration callback's private adaptive/diagnostic context; this callback
|
||||
/// receives the opaque `ZSTD_CCtx` directly and delegates to Rust's public
|
||||
/// `ZSTD_compressStream2` implementation.
|
||||
#[cfg(not(test))]
|
||||
#[cfg(all(feature = "compression", not(test)))]
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn FIO_rust_zstd_compressStream(
|
||||
cctx: *mut c_void,
|
||||
@@ -4935,6 +4944,7 @@ pub unsafe extern "C" fn FIO_rust_compressLzmaFrame(
|
||||
/// parameters so the C-owned `dRess_t` never crosses into Rust. The input
|
||||
/// pool is advanced only after a successful decoder call; in particular, an
|
||||
/// error leaves the current input buffer untouched for `FIO_zstdErrorHelp()`.
|
||||
#[cfg(feature = "decompression")]
|
||||
unsafe fn decompress_zstd_frame_with(
|
||||
f_ctx: *mut c_void,
|
||||
dctx: *mut c_void,
|
||||
@@ -5030,6 +5040,7 @@ unsafe fn decompress_zstd_frame_with(
|
||||
/// leaves a non-zstd header or a short trailing buffer untouched for C's
|
||||
/// mixed-format dispatcher. Completed-frame output remains accounted for if
|
||||
/// a later frame reports an error.
|
||||
#[cfg(feature = "decompression")]
|
||||
unsafe fn decompress_zstd_frames_with(
|
||||
f_ctx: *mut c_void,
|
||||
dctx: *mut c_void,
|
||||
@@ -5098,6 +5109,7 @@ unsafe fn decompress_zstd_frames_with(
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
unsafe extern "C" fn fio_zstd_reset(dctx: *mut c_void, reset: c_int) -> usize {
|
||||
unsafe {
|
||||
crate::zstd_decompress::ZSTD_DCtx_reset(
|
||||
@@ -5107,6 +5119,7 @@ unsafe extern "C" fn fio_zstd_reset(dctx: *mut c_void, reset: c_int) -> usize {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
unsafe extern "C" fn fio_zstd_decompress(
|
||||
dctx: *mut c_void,
|
||||
output: *mut crate::zstd_decompress::ZSTD_outBuffer,
|
||||
@@ -5122,6 +5135,7 @@ unsafe extern "C" fn fio_zstd_decompress(
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
#[cfg(feature = "decompression")]
|
||||
pub unsafe extern "C" fn FIO_rust_decompressZstdFrame(
|
||||
f_ctx: *mut c_void,
|
||||
dctx: *mut c_void,
|
||||
@@ -5152,6 +5166,7 @@ pub unsafe extern "C" fn FIO_rust_decompressZstdFrame(
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
#[cfg(feature = "decompression")]
|
||||
pub unsafe extern "C" fn FIO_rust_decompressZstdFrames(
|
||||
f_ctx: *mut c_void,
|
||||
dctx: *mut c_void,
|
||||
@@ -5182,9 +5197,11 @@ pub unsafe extern "C" fn FIO_rust_decompressZstdFrames(
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
const FIO_ERROR_FRAME_DECODING: u64 = u64::MAX - 1;
|
||||
|
||||
#[inline]
|
||||
#[cfg(feature = "decompression")]
|
||||
unsafe fn zstd_frame_policy_result(
|
||||
callback_context: *mut c_void,
|
||||
src_file_name: *const c_char,
|
||||
@@ -5211,6 +5228,7 @@ unsafe fn zstd_frame_policy_result(
|
||||
/// Run the default zstd-frame loop and apply its historical result/error
|
||||
/// policy. C retains only exact diagnostics and private resource callbacks.
|
||||
#[no_mangle]
|
||||
#[cfg(feature = "decompression")]
|
||||
pub unsafe extern "C" fn FIO_rust_decompressZstdFramePolicy(
|
||||
state: *const FIO_rust_zstd_frame_policy_state,
|
||||
) -> u64 {
|
||||
@@ -5797,14 +5815,14 @@ where
|
||||
|
||||
#[inline]
|
||||
unsafe fn is_zstd_frame_for_dispatch(buffer: &[u8]) -> bool {
|
||||
#[cfg(test)]
|
||||
#[cfg(any(test, not(feature = "decompression")))]
|
||||
{
|
||||
/* Standalone Rust tests do not link the C legacy-decoder shim. The
|
||||
* dispatch tests only need the modern frame magic; production keeps
|
||||
* the complete public predicate below. */
|
||||
buffer.starts_with(&[0x28, 0xB5, 0x2F, 0xFD])
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
#[cfg(all(feature = "decompression", not(test)))]
|
||||
{
|
||||
unsafe { crate::zstd_decompress::ZSTD_isFrame(buffer.as_ptr().cast(), buffer.len()) != 0 }
|
||||
}
|
||||
|
||||
+18
-1
@@ -17,6 +17,20 @@ GZFILES = gzclose.o gzlib.o gzread.o gzwrite.o
|
||||
EXAMPLE_PATH = examples
|
||||
PROGRAMS_PATH = ../programs
|
||||
TEST_FILE = ../doc/zstd_compression_format.md
|
||||
CARGO ?= cargo
|
||||
RUST_DIR := ../rust
|
||||
RUST_CLI_DIR := $(RUST_DIR)/cli
|
||||
RUST_CLI_MANIFEST := $(RUST_CLI_DIR)/Cargo.toml
|
||||
RUST_CLI_HELPER_SOURCES := $(RUST_CLI_MANIFEST) $(RUST_CLI_DIR)/Cargo.lock \
|
||||
$(RUST_CLI_DIR)/src/lib.rs \
|
||||
$(RUST_DIR)/src/timefn.rs $(RUST_DIR)/src/benchfn.rs \
|
||||
$(RUST_DIR)/src/datagen.rs $(RUST_DIR)/src/lorem.rs \
|
||||
$(RUST_DIR)/src/util.rs
|
||||
RUST_CLI_HELPERS_TARGET_DIR := $(RUST_DIR)/target/cli-helpers
|
||||
RUST_CLI_HELPERS_STATICLIB := $(RUST_CLI_HELPERS_TARGET_DIR)/release/libzstd_cli_rs.a
|
||||
RUST_CLI_HELPERS_CARGO_FLAGS := --manifest-path $(RUST_CLI_MANIFEST) --release \
|
||||
--target-dir $(RUST_CLI_HELPERS_TARGET_DIR) \
|
||||
--no-default-features --features helpers
|
||||
|
||||
vpath %.c $(PROGRAMS_PATH) $(EXAMPLE_PATH) $(ZLIBWRAPPER_PATH)
|
||||
|
||||
@@ -98,7 +112,7 @@ fitblk: fitblk.o zstd_zlibwrapper.o $(ZSTDLIBRARY)
|
||||
fitblk_zstd: fitblk.o zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY)
|
||||
$(LINK.o) $^ $(LDLIBS) $(OUTPUT_OPTION)
|
||||
|
||||
zwrapbench: zwrapbench.o zstd_zlibwrapper.o util.o timefn.o datagen.o $(ZSTDLIBRARY)
|
||||
zwrapbench: zwrapbench.o zstd_zlibwrapper.o util.o timefn.o datagen.o $(ZSTDLIBRARY) $(RUST_CLI_HELPERS_STATICLIB)
|
||||
|
||||
|
||||
zstd_zlibwrapper.o: zstd_zlibwrapper.h
|
||||
@@ -111,6 +125,9 @@ zstdTurnedOn_zlibwrapper.o: zstd_zlibwrapper.c zstd_zlibwrapper.h
|
||||
$(ZSTDLIBRARY):
|
||||
$(MAKE) -C $(ZSTDLIBDIR) libzstd.a
|
||||
|
||||
$(RUST_CLI_HELPERS_STATICLIB): $(RUST_CLI_HELPER_SOURCES)
|
||||
$(CARGO) build $(RUST_CLI_HELPERS_CARGO_FLAGS)
|
||||
|
||||
$(ZSTDLIBDIR)/libzstd.so:
|
||||
$(MAKE) -C $(ZSTDLIBDIR) libzstd
|
||||
|
||||
|
||||
Reference in New Issue
Block a user