diff --git a/lib/Makefile b/lib/Makefile index a6a0eb09d..3517d00fb 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -48,6 +48,82 @@ ifneq ($(ZSTD_LIB_DICTBUILDER), 0) ZSTD_FILES += $(ZSTD_DICTBUILDER_FILES) endif +# The migrated implementation is built as a Rust static archive. The archive +# is feature-specific so a build forced to use a particular HUF decoder never +# reuses the default decoder implementation. RUST_TARGET can be set for a +# cross build; -m32 selects the supported 32-bit Linux target automatically. +CARGO ?= cargo +RUST_DIR ?= ../rust +RUST_MANIFEST := $(RUST_DIR)/Cargo.toml +RUST_SOURCES := $(RUST_MANIFEST) $(RUST_DIR)/Cargo.lock \ + $(shell find $(RUST_DIR)/src -type f -name '*.rs' -print) + +RUST_HUF_FORCE_X1 := $(HUF_FORCE_DECOMPRESS_X1) +RUST_HUF_FORCE_X2 := $(HUF_FORCE_DECOMPRESS_X2) +ifneq ($(filter -DHUF_FORCE_DECOMPRESS_X1 -DHUF_FORCE_DECOMPRESS_X1=%,$(CPPFLAGS) $(CFLAGS) $(MOREFLAGS)),) +RUST_HUF_FORCE_X1 := 1 +endif +ifneq ($(filter -DHUF_FORCE_DECOMPRESS_X2 -DHUF_FORCE_DECOMPRESS_X2=%,$(CPPFLAGS) $(CFLAGS) $(MOREFLAGS)),) +RUST_HUF_FORCE_X2 := 1 +endif + +ifneq ($(RUST_HUF_FORCE_X1),0) +ifneq ($(RUST_HUF_FORCE_X2),0) +$(error HUF_FORCE_DECOMPRESS_X1 and HUF_FORCE_DECOMPRESS_X2 are mutually exclusive) +endif +endif + +empty := +space := $(empty) $(empty) +comma := , + +RUST_CARGO_FEATURES := +ifneq ($(ZSTD_LIB_COMPRESSION),0) +RUST_CARGO_FEATURES += compression +endif +ifneq ($(ZSTD_LIB_DECOMPRESSION),0) +RUST_CARGO_FEATURES += decompression +endif + +RUST_HUF_FEATURE := +RUST_BUILD_CONFIG := c$(ZSTD_LIB_COMPRESSION)-d$(ZSTD_LIB_DECOMPRESSION)-default +ifneq ($(RUST_HUF_FORCE_X1),0) +RUST_HUF_FEATURE := huf-force-decompress-x1 +RUST_BUILD_CONFIG := c$(ZSTD_LIB_COMPRESSION)-d$(ZSTD_LIB_DECOMPRESSION)-huf-force-decompress-x1 +endif +ifneq ($(RUST_HUF_FORCE_X2),0) +RUST_HUF_FEATURE := huf-force-decompress-x2 +RUST_BUILD_CONFIG := c$(ZSTD_LIB_COMPRESSION)-d$(ZSTD_LIB_DECOMPRESSION)-huf-force-decompress-x2 +endif +ifneq ($(RUST_HUF_FEATURE),) +ifneq ($(ZSTD_LIB_DECOMPRESSION),0) +RUST_CARGO_FEATURES += $(RUST_HUF_FEATURE) +endif +endif +RUST_CARGO_FEATURES := $(subst $(space),$(comma),$(strip $(RUST_CARGO_FEATURES))) + +RUST_TARGET ?= +ifeq ($(strip $(RUST_TARGET)),) +ifneq ($(filter -m32,$(CC) $(CPPFLAGS) $(CFLAGS) $(MOREFLAGS)),) +RUST_TARGET := i686-unknown-linux-gnu +endif +endif + +RUST_TARGET_DIR := $(RUST_DIR)/target/$(RUST_BUILD_CONFIG) +RUST_CARGO_FLAGS := --manifest-path $(RUST_MANIFEST) --release \ + --target-dir $(RUST_TARGET_DIR) --no-default-features +ifneq ($(RUST_CARGO_FEATURES),) +RUST_CARGO_FLAGS += --features $(RUST_CARGO_FEATURES) +endif +ifneq ($(strip $(RUST_TARGET)),) +RUST_CARGO_FLAGS += --target $(RUST_TARGET) +RUST_TARGET_COMPONENT := /$(RUST_TARGET) +endif +RUST_STATICLIB := $(RUST_TARGET_DIR)$(RUST_TARGET_COMPONENT)/release/libzstd_rs.a + +$(RUST_STATICLIB): $(RUST_SOURCES) + $(CARGO) build $(RUST_CARGO_FLAGS) + ZSTD_LOCAL_SRC := $(notdir $(ZSTD_FILES)) ZSTD_LOCAL_OBJ0 := $(ZSTD_LOCAL_SRC:.c=.o) ZSTD_LOCAL_OBJ := $(ZSTD_LOCAL_OBJ0:.S=.o) @@ -91,6 +167,15 @@ else SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER) endif +# A dynamic library must retain every migrated public Rust export, including +# functions no C object calls directly. GNU-like linkers need an explicit +# whole-archive region for that; Darwin spells the same operation force_load. +ifeq ($(UNAME_TARGET_SYSTEM), Darwin) +RUST_DYNLIB_LINK = -Wl,-force_load,$(abspath $(RUST_STATICLIB)) +else +RUST_DYNLIB_LINK = -Wl,--whole-archive $(RUST_STATICLIB) -Wl,--no-whole-archive +endif + .PHONY: all all: lib @@ -118,14 +203,20 @@ else ZSTD_STATICLIB_DIR := $(BUILD_DIR)/static ZSTD_STATICLIB := $(ZSTD_STATICLIB_DIR)/libzstd.a ZSTD_STATICLIB_OBJ := $(addprefix $(ZSTD_STATICLIB_DIR)/,$(ZSTD_LOCAL_OBJ)) +ZSTD_STATICLIB_RUST_DIR := $(ZSTD_STATICLIB_DIR)/rust $(ZSTD_STATICLIB): ARFLAGS = rcs $(ZSTD_STATICLIB): | $(ZSTD_STATICLIB_DIR) -$(ZSTD_STATICLIB): $(ZSTD_STATICLIB_OBJ) +$(ZSTD_STATICLIB): $(ZSTD_STATICLIB_OBJ) $(RUST_STATICLIB) # Check for multithread flag at target execution time $(if $(filter -DZSTD_MULTITHREAD,$(CPPFLAGS)),\ @echo compiling multi-threaded static library $(LIBVER),\ @echo compiling single-threaded static library $(LIBVER)) - $(AR) $(ARFLAGS) $@ $^ + $(RM) -rf $(ZSTD_STATICLIB_RUST_DIR) + $(MKDIR) $(ZSTD_STATICLIB_RUST_DIR) + (cd $(ZSTD_STATICLIB_RUST_DIR) && $(AR) x $(abspath $(RUST_STATICLIB))) + $(RM) -f $@ + $(AR) $(ARFLAGS) $@ $(ZSTD_STATICLIB_OBJ) $(ZSTD_STATICLIB_RUST_DIR)/*.o + $(RM) -rf $(ZSTD_STATICLIB_RUST_DIR) libzstd.a: $(ZSTD_STATICLIB) cp -f $< $@ @@ -135,9 +226,9 @@ endif ifneq (,$(filter Windows%,$(TARGET_SYSTEM))) LIBZSTD = dll/libzstd.dll -$(LIBZSTD): $(ZSTD_FILES) +$(LIBZSTD): $(ZSTD_FILES) $(RUST_STATICLIB) @echo compiling dynamic library $(LIBVER) - $(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll/libzstd.dll.a -shared $^ -o $@ + $(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll/libzstd.dll.a -shared $(ZSTD_FILES) $(RUST_DYNLIB_LINK) -o $@ else # not Windows @@ -161,12 +252,12 @@ ZSTD_DYNLIB := $(ZSTD_DYNLIB_DIR)/$(LIBZSTD) ZSTD_DYNLIB_OBJ := $(addprefix $(ZSTD_DYNLIB_DIR)/,$(ZSTD_LOCAL_OBJ)) $(ZSTD_DYNLIB): | $(ZSTD_DYNLIB_DIR) -$(ZSTD_DYNLIB): $(ZSTD_DYNLIB_OBJ) +$(ZSTD_DYNLIB): $(ZSTD_DYNLIB_OBJ) $(RUST_STATICLIB) # Check for multithread flag at target execution time $(if $(filter -DZSTD_MULTITHREAD,$(CPPFLAGS)),\ @echo compiling multi-threaded dynamic library $(LIBVER),\ @echo compiling single-threaded dynamic library $(LIBVER)) - $(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@ + $(CC) $(FLAGS) $(ZSTD_DYNLIB_OBJ) $(RUST_DYNLIB_LINK) $(LDFLAGS) $(SONAME_FLAGS) -o $@ @echo creating versioned links ln -sf $@ libzstd.$(SHARED_EXT_MAJOR) ln -sf $@ libzstd.$(SHARED_EXT) @@ -248,14 +339,14 @@ ZSTDMT_FILES = zstdmt_compress.c ZSTD_NOMT_FILES = $(filter-out $(ZSTDMT_FILES),$(notdir $(ZSTD_FILES))) libzstd-nomt: CFLAGS += -fPIC -fvisibility=hidden libzstd-nomt: LDFLAGS += -shared -libzstd-nomt: $(ZSTD_NOMT_FILES) +libzstd-nomt: $(ZSTD_NOMT_FILES) $(RUST_STATICLIB) @echo compiling single-thread dynamic library $(LIBVER) @echo files : $(ZSTD_NOMT_FILES) @if echo "$(ZSTD_NOMT_FILES)" | tr ' ' '\n' | $(GREP) -q zstdmt; then \ echo "Error: Found zstdmt in list."; \ exit 1; \ fi - $(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@ + $(CC) $(FLAGS) $(ZSTD_NOMT_FILES) $(RUST_DYNLIB_LINK) $(LDFLAGS) $(SONAME_FLAGS) -o $@ .PHONY: clean clean: diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 443b949c2..65d06c2e3 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -7,7 +7,9 @@ edition = "2021" crate-type = ["staticlib"] [features] -default = [] +default = ["compression", "decompression"] +compression = [] +decompression = [] huf-force-decompress-x1 = [] huf-force-decompress-x2 = [] diff --git a/rust/README.md b/rust/README.md index f19f4990f..a2a5b6785 100644 --- a/rust/README.md +++ b/rust/README.md @@ -46,9 +46,12 @@ source file whose implementation has moved to Rust remains in the original makefile source list as a small shim so header configuration and platform preprocessor behavior stay available during the transition. -The test and program makefiles select an archive directory for the active C -configuration: default, forced HUF X1/X2, and the matching Rust target for -32-bit C binaries. When the HUF mode changes, they also rebuild cached C +The library, test, and program makefiles select an archive directory for the +active C configuration: enabled compression/decompression modules, default or +forced HUF X1/X2, and the matching Rust target for 32-bit C binaries. The +native static archive flattens Rust object members rather than nesting a Rust +archive, while the native shared library retains all migrated Rust exports. +When the HUF mode changes, the test and program paths also rebuild cached C outputs before linking. This prevents original C tests from using a stale or configuration-incompatible implementation. @@ -69,6 +72,7 @@ the narrow target for the component being migrated. For example: ```sh make -C tests fuzzer ./tests/fuzzer -i1 --no-big-tests +make -C tests test-rust-lib-smoke ``` Broader `tests/Makefile` targets remain the authoritative integration gates as diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 7a8bffecf..dc3325213 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -7,14 +7,19 @@ pub mod cpu; pub mod debug; pub mod entropy_common; pub mod errors; +#[cfg(feature = "compression")] pub mod fse_compress; pub mod fse_decompress; +#[cfg(feature = "compression")] pub mod hist; +#[cfg(feature = "decompression")] pub mod huf_decompress; pub mod mem; pub mod pool; pub mod threading; pub mod xxhash; pub mod zstd_common; +#[cfg(feature = "decompression")] pub mod zstd_ddict; +#[cfg(feature = "compression")] pub mod zstd_presplit; diff --git a/tests/.gitignore b/tests/.gitignore index 311a8b5eb..3a535337a 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -18,6 +18,7 @@ datagen paramgrill paramgrill32 roundTripCrash +rustLibSmoke longmatch symbols legacy diff --git a/tests/Makefile b/tests/Makefile index 3a4fd8735..e9f4ac553 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -308,6 +308,13 @@ largeDictionary: $(ZSTDMT_OBJECTS) $(PRGDIR)/datagen.c largeDictionary.c CLEAN += invalidDictionaries invalidDictionaries : $(ZSTD_OBJECTS) invalidDictionaries.c +# Exercise the public static archive as an external C consumer would. This +# catches a nested Rust archive or unresolved C/Rust cycle that direct object +# links cannot expose. +CLEAN += rustLibSmoke +rustLibSmoke : $(LIB_BINDIR)/libzstd.a rustLibSmoke.c + $(LINK.c) $(filter %.c,$^) $(filter %.a,$^) -o $@$(EXT) + CLEAN += legacy legacy : CPPFLAGS += -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=4 legacy : $(ZSTD_FILES) legacy.c @@ -413,7 +420,7 @@ check: test-zstd fuzztest: test-fuzzer test-zstream test-decodecorpus .PHONY: test -test: test-zstd test-cli-tests test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus +test: test-zstd test-cli-tests test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-rust-lib-smoke test-legacy test-decodecorpus ifeq ($(QEMU_SYS),) test: test-pool endif @@ -491,6 +498,10 @@ test-largeDictionary: largeDictionary test-invalidDictionaries: invalidDictionaries $(QEMU_SYS) ./invalidDictionaries +.PHONY: test-rust-lib-smoke +test-rust-lib-smoke: rustLibSmoke + $(QEMU_SYS) ./rustLibSmoke + test-legacy: legacy $(QEMU_SYS) ./legacy diff --git a/tests/rustLibSmoke.c b/tests/rustLibSmoke.c new file mode 100644 index 000000000..5a246d3b7 --- /dev/null +++ b/tests/rustLibSmoke.c @@ -0,0 +1,66 @@ +/* + * Link and run through the installed-style static archive, rather than the + * direct C-object-plus-Rust-archive test path used by the other test tools. + */ +#include +#include +#include + +#include "zstd.h" + +static int checkError(size_t result, const char* operation) +{ + if (!ZSTD_isError(result)) return 0; + fprintf(stderr, "%s: %s\n", operation, ZSTD_getErrorName(result)); + return 1; +} + +int main(void) +{ + enum { sourceSize = 64 * 1024 }; + unsigned char source[sourceSize]; + size_t const compressedCapacity = ZSTD_compressBound(sourceSize); + void* compressed = malloc(compressedCapacity); + void* restored = malloc(sourceSize); + size_t compressedSize; + size_t restoredSize; + ZSTD_DDict* ddict; + size_t index; + + if (compressed == NULL || restored == NULL) { + fputs("allocation failed\n", stderr); + free(restored); + free(compressed); + return 1; + } + + for (index = 0; index < sourceSize; ++index) { + source[index] = (unsigned char)((index * 13 + index / 97) & 0x1F); + } + compressedSize = ZSTD_compress(compressed, compressedCapacity, + source, sourceSize, 3); + if (checkError(compressedSize, "ZSTD_compress")) goto cleanup; + + restoredSize = ZSTD_decompress(restored, sourceSize, compressed, compressedSize); + if (checkError(restoredSize, "ZSTD_decompress")) goto cleanup; + if (restoredSize != sourceSize || memcmp(source, restored, sourceSize) != 0) { + fputs("round trip mismatch\n", stderr); + goto cleanup; + } + + /* This must pull the Rust DDict object and resolve its C helper cycle. */ + ddict = ZSTD_createDDict(NULL, 0); + if (ddict == NULL || ZSTD_freeDDict(ddict) != 0) { + fputs("DDict lifecycle failed\n", stderr); + goto cleanup; + } + + free(restored); + free(compressed); + return 0; + +cleanup: + free(restored); + free(compressed); + return 1; +}