ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

build(rust): link migrated code into native libraries

Browse Source 
Build feature-matched Rust archives for the native Make library path. Flatten
their members into libzstd.a so static consumers resolve C-to-Rust and
Rust-to-C references in one archive, and whole-archive link shared outputs so
every migrated public ABI export remains available.

Rust compression and decompression features now mirror the C partial-library
switches. This prevents compression-only artifacts from retaining DDict
references to decompression-only C helpers.

Add a C archive smoke test that round-trips data and exercises the DDict
lifecycle through lib/libzstd.a, rather than direct test-object links.

Test Plan:
- cargo fmt, strict Clippy, cargo test --all-targets, release build
- Rust checks with compression-only, decompression-only, and no features
- full, partial, forced-HUF, and 32-bit static/shared Make library probes
- test-rust-lib-smoke, fuzzer, zstreamtest, and invalidDictionaries

Refs: rust/README.md
This commit is contained in:
ddidderr
2026-07-10 21:41:22 +02:00
parent 24d01b92fb
commit d784406374
7 changed files with 193 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/Makefile
+99 -8
View File
@@ -48,6 +48,82 @@ ifneq ($(ZSTD_LIB_DICTBUILDER), 0)
ZSTD_FILES += $(ZSTD_DICTBUILDER_FILES)
endif
# The migrated implementation is built as a Rust static archive. The archive
# is feature-specific so a build forced to use a particular HUF decoder never
# reuses the default decoder implementation. RUST_TARGET can be set for a
# cross build; -m32 selects the supported 32-bit Linux target automatically.
CARGO ?= cargo
RUST_DIR ?= ../rust
RUST_MANIFEST := $(RUST_DIR)/Cargo.toml
RUST_SOURCES := $(RUST_MANIFEST) $(RUST_DIR)/Cargo.lock \
$(shell find $(RUST_DIR)/src -type f -name '*.rs' -print)
RUST_HUF_FORCE_X1 := $(HUF_FORCE_DECOMPRESS_X1)
RUST_HUF_FORCE_X2 := $(HUF_FORCE_DECOMPRESS_X2)
ifneq ($(filter -DHUF_FORCE_DECOMPRESS_X1 -DHUF_FORCE_DECOMPRESS_X1=%,$(CPPFLAGS) $(CFLAGS) $(MOREFLAGS)),)
RUST_HUF_FORCE_X1 := 1
endif
ifneq ($(filter -DHUF_FORCE_DECOMPRESS_X2 -DHUF_FORCE_DECOMPRESS_X2=%,$(CPPFLAGS) $(CFLAGS) $(MOREFLAGS)),)
RUST_HUF_FORCE_X2 := 1
endif
ifneq ($(RUST_HUF_FORCE_X1),0)
ifneq ($(RUST_HUF_FORCE_X2),0)
$(error HUF_FORCE_DECOMPRESS_X1 and HUF_FORCE_DECOMPRESS_X2 are mutually exclusive)
endif
endif
empty :=
space := $(empty) $(empty)
comma := ,
RUST_CARGO_FEATURES :=
ifneq ($(ZSTD_LIB_COMPRESSION),0)
RUST_CARGO_FEATURES += compression
endif
ifneq ($(ZSTD_LIB_DECOMPRESSION),0)
RUST_CARGO_FEATURES += decompression
endif
RUST_HUF_FEATURE :=
RUST_BUILD_CONFIG := c$(ZSTD_LIB_COMPRESSION)-d$(ZSTD_LIB_DECOMPRESSION)-default
ifneq ($(RUST_HUF_FORCE_X1),0)
RUST_HUF_FEATURE := huf-force-decompress-x1
RUST_BUILD_CONFIG := c$(ZSTD_LIB_COMPRESSION)-d$(ZSTD_LIB_DECOMPRESSION)-huf-force-decompress-x1
endif
ifneq ($(RUST_HUF_FORCE_X2),0)
RUST_HUF_FEATURE := huf-force-decompress-x2
RUST_BUILD_CONFIG := c$(ZSTD_LIB_COMPRESSION)-d$(ZSTD_LIB_DECOMPRESSION)-huf-force-decompress-x2
endif
ifneq ($(RUST_HUF_FEATURE),)
ifneq ($(ZSTD_LIB_DECOMPRESSION),0)
RUST_CARGO_FEATURES += $(RUST_HUF_FEATURE)
endif
endif
RUST_CARGO_FEATURES := $(subst $(space),$(comma),$(strip $(RUST_CARGO_FEATURES)))
RUST_TARGET ?=
ifeq ($(strip $(RUST_TARGET)),)
ifneq ($(filter -m32,$(CC) $(CPPFLAGS) $(CFLAGS) $(MOREFLAGS)),)
RUST_TARGET := i686-unknown-linux-gnu
endif
endif
RUST_TARGET_DIR := $(RUST_DIR)/target/$(RUST_BUILD_CONFIG)
RUST_CARGO_FLAGS := --manifest-path $(RUST_MANIFEST) --release \
--target-dir $(RUST_TARGET_DIR) --no-default-features
ifneq ($(RUST_CARGO_FEATURES),)
RUST_CARGO_FLAGS += --features $(RUST_CARGO_FEATURES)
endif
ifneq ($(strip $(RUST_TARGET)),)
RUST_CARGO_FLAGS += --target $(RUST_TARGET)
RUST_TARGET_COMPONENT := /$(RUST_TARGET)
endif
RUST_STATICLIB := $(RUST_TARGET_DIR)$(RUST_TARGET_COMPONENT)/release/libzstd_rs.a
$(RUST_STATICLIB): $(RUST_SOURCES)
$(CARGO) build $(RUST_CARGO_FLAGS)
ZSTD_LOCAL_SRC := $(notdir $(ZSTD_FILES))
ZSTD_LOCAL_OBJ0 := $(ZSTD_LOCAL_SRC:.c=.o)
ZSTD_LOCAL_OBJ := $(ZSTD_LOCAL_OBJ0:.S=.o)
@@ -91,6 +167,15 @@ else
SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
endif
# A dynamic library must retain every migrated public Rust export, including
# functions no C object calls directly. GNU-like linkers need an explicit
# whole-archive region for that; Darwin spells the same operation force_load.
ifeq ($(UNAME_TARGET_SYSTEM), Darwin)
RUST_DYNLIB_LINK = -Wl,-force_load,$(abspath $(RUST_STATICLIB))
else
RUST_DYNLIB_LINK = -Wl,--whole-archive $(RUST_STATICLIB) -Wl,--no-whole-archive
endif
.PHONY: all
all: lib
@@ -118,14 +203,20 @@ else
ZSTD_STATICLIB_DIR := $(BUILD_DIR)/static
ZSTD_STATICLIB := $(ZSTD_STATICLIB_DIR)/libzstd.a
ZSTD_STATICLIB_OBJ := $(addprefix $(ZSTD_STATICLIB_DIR)/,$(ZSTD_LOCAL_OBJ))
ZSTD_STATICLIB_RUST_DIR := $(ZSTD_STATICLIB_DIR)/rust
$(ZSTD_STATICLIB): ARFLAGS = rcs
$(ZSTD_STATICLIB): | $(ZSTD_STATICLIB_DIR)
$(ZSTD_STATICLIB): $(ZSTD_STATICLIB_OBJ)
$(ZSTD_STATICLIB): $(ZSTD_STATICLIB_OBJ) $(RUST_STATICLIB)
# Check for multithread flag at target execution time
$(if $(filter -DZSTD_MULTITHREAD,$(CPPFLAGS)),\
@echo compiling multi-threaded static library $(LIBVER),\
@echo compiling single-threaded static library $(LIBVER))
$(AR) $(ARFLAGS) $@ $^
$(RM) -rf $(ZSTD_STATICLIB_RUST_DIR)
$(MKDIR) $(ZSTD_STATICLIB_RUST_DIR)
(cd $(ZSTD_STATICLIB_RUST_DIR) && $(AR) x $(abspath $(RUST_STATICLIB)))
$(RM) -f $@
$(AR) $(ARFLAGS) $@ $(ZSTD_STATICLIB_OBJ) $(ZSTD_STATICLIB_RUST_DIR)/*.o
$(RM) -rf $(ZSTD_STATICLIB_RUST_DIR)
libzstd.a: $(ZSTD_STATICLIB)
cp -f $< $@
@@ -135,9 +226,9 @@ endif
ifneq (,$(filter Windows%,$(TARGET_SYSTEM)))
LIBZSTD = dll/libzstd.dll
$(LIBZSTD): $(ZSTD_FILES)
$(LIBZSTD): $(ZSTD_FILES) $(RUST_STATICLIB)
@echo compiling dynamic library $(LIBVER)
$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll/libzstd.dll.a -shared $^ -o $@
$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll/libzstd.dll.a -shared $(ZSTD_FILES) $(RUST_DYNLIB_LINK) -o $@
else # not Windows
@@ -161,12 +252,12 @@ ZSTD_DYNLIB := $(ZSTD_DYNLIB_DIR)/$(LIBZSTD)
ZSTD_DYNLIB_OBJ := $(addprefix $(ZSTD_DYNLIB_DIR)/,$(ZSTD_LOCAL_OBJ))
$(ZSTD_DYNLIB): | $(ZSTD_DYNLIB_DIR)
$(ZSTD_DYNLIB): $(ZSTD_DYNLIB_OBJ)
$(ZSTD_DYNLIB): $(ZSTD_DYNLIB_OBJ) $(RUST_STATICLIB)
# Check for multithread flag at target execution time
$(if $(filter -DZSTD_MULTITHREAD,$(CPPFLAGS)),\
@echo compiling multi-threaded dynamic library $(LIBVER),\
@echo compiling single-threaded dynamic library $(LIBVER))
$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
$(CC) $(FLAGS) $(ZSTD_DYNLIB_OBJ) $(RUST_DYNLIB_LINK) $(LDFLAGS) $(SONAME_FLAGS) -o $@
@echo creating versioned links
ln -sf $@ libzstd.$(SHARED_EXT_MAJOR)
ln -sf $@ libzstd.$(SHARED_EXT)
@@ -248,14 +339,14 @@ ZSTDMT_FILES = zstdmt_compress.c
ZSTD_NOMT_FILES = $(filter-out $(ZSTDMT_FILES),$(notdir $(ZSTD_FILES)))
libzstd-nomt: CFLAGS += -fPIC -fvisibility=hidden
libzstd-nomt: LDFLAGS += -shared
libzstd-nomt: $(ZSTD_NOMT_FILES)
libzstd-nomt: $(ZSTD_NOMT_FILES) $(RUST_STATICLIB)
@echo compiling single-thread dynamic library $(LIBVER)
@echo files : $(ZSTD_NOMT_FILES)
@if echo "$(ZSTD_NOMT_FILES)" | tr ' ' '\n' | $(GREP) -q zstdmt; then \
echo "Error: Found zstdmt in list."; \
exit 1; \
fi
$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
$(CC) $(FLAGS) $(ZSTD_NOMT_FILES) $(RUST_DYNLIB_LINK) $(LDFLAGS) $(SONAME_FLAGS) -o $@
.PHONY: clean
clean:
rust/Cargo.toml
+3 -1
View File
@@ -7,7 +7,9 @@ edition = "2021"
crate-type = ["staticlib"]
[features]
default = []
default = ["compression", "decompression"]
compression = []
decompression = []
huf-force-decompress-x1 = []
huf-force-decompress-x2 = []
rust/README.md
+7 -3
View File
@@ -46,9 +46,12 @@ source file whose implementation has moved to Rust remains in the original
makefile source list as a small shim so header configuration and platform
preprocessor behavior stay available during the transition.
The test and program makefiles select an archive directory for the active C
configuration: default, forced HUF X1/X2, and the matching Rust target for
32-bit C binaries. When the HUF mode changes, they also rebuild cached C
The library, test, and program makefiles select an archive directory for the
active C configuration: enabled compression/decompression modules, default or
forced HUF X1/X2, and the matching Rust target for 32-bit C binaries. The
native static archive flattens Rust object members rather than nesting a Rust
archive, while the native shared library retains all migrated Rust exports.
When the HUF mode changes, the test and program paths also rebuild cached C
outputs before linking. This prevents original C tests from using a stale or
configuration-incompatible implementation.
@@ -69,6 +72,7 @@ the narrow target for the component being migrated. For example:
```sh
make -C tests fuzzer
./tests/fuzzer -i1 --no-big-tests
make -C tests test-rust-lib-smoke
```
Broader `tests/Makefile` targets remain the authoritative integration gates as
rust/src/lib.rs
+5
View File
@@ -7,14 +7,19 @@ pub mod cpu;
pub mod debug;
pub mod entropy_common;
pub mod errors;
#[cfg(feature = "compression")]
pub mod fse_compress;
pub mod fse_decompress;
#[cfg(feature = "compression")]
pub mod hist;
#[cfg(feature = "decompression")]
pub mod huf_decompress;
pub mod mem;
pub mod pool;
pub mod threading;
pub mod xxhash;
pub mod zstd_common;
#[cfg(feature = "decompression")]
pub mod zstd_ddict;
#[cfg(feature = "compression")]
pub mod zstd_presplit;
tests/.gitignore
+1
View File
@@ -18,6 +18,7 @@ datagen
paramgrill
paramgrill32
roundTripCrash
rustLibSmoke
longmatch
symbols
legacy
tests/Makefile
+12 -1
View File
@@ -308,6 +308,13 @@ largeDictionary: $(ZSTDMT_OBJECTS) $(PRGDIR)/datagen.c largeDictionary.c
CLEAN += invalidDictionaries
invalidDictionaries : $(ZSTD_OBJECTS) invalidDictionaries.c
# Exercise the public static archive as an external C consumer would. This
# catches a nested Rust archive or unresolved C/Rust cycle that direct object
# links cannot expose.
CLEAN += rustLibSmoke
rustLibSmoke : $(LIB_BINDIR)/libzstd.a rustLibSmoke.c
$(LINK.c) $(filter %.c,$^) $(filter %.a,$^) -o $@$(EXT)
CLEAN += legacy
legacy : CPPFLAGS += -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=4
legacy : $(ZSTD_FILES) legacy.c
@@ -413,7 +420,7 @@ check: test-zstd
fuzztest: test-fuzzer test-zstream test-decodecorpus
.PHONY: test
test: test-zstd test-cli-tests test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus
test: test-zstd test-cli-tests test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-rust-lib-smoke test-legacy test-decodecorpus
ifeq ($(QEMU_SYS),)
test: test-pool
endif
@@ -491,6 +498,10 @@ test-largeDictionary: largeDictionary
test-invalidDictionaries: invalidDictionaries
$(QEMU_SYS) ./invalidDictionaries
.PHONY: test-rust-lib-smoke
test-rust-lib-smoke: rustLibSmoke
$(QEMU_SYS) ./rustLibSmoke
test-legacy: legacy
$(QEMU_SYS) ./legacy
tests/rustLibSmoke.c
+66
View File
@@ -0,0 +1,66 @@
/*
* Link and run through the installed-style static archive, rather than the
* direct C-object-plus-Rust-archive test path used by the other test tools.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "zstd.h"
static int checkError(size_t result, const char* operation)
{
if (!ZSTD_isError(result)) return 0;
fprintf(stderr, "%s: %s\n", operation, ZSTD_getErrorName(result));
return 1;
}
int main(void)
{
enum { sourceSize = 64 * 1024 };
unsigned char source[sourceSize];
size_t const compressedCapacity = ZSTD_compressBound(sourceSize);
void* compressed = malloc(compressedCapacity);
void* restored = malloc(sourceSize);
size_t compressedSize;
size_t restoredSize;
ZSTD_DDict* ddict;
size_t index;
if (compressed == NULL || restored == NULL) {
fputs("allocation failed\n", stderr);
free(restored);
free(compressed);
return 1;
}
for (index = 0; index < sourceSize; ++index) {
source[index] = (unsigned char)((index * 13 + index / 97) & 0x1F);
}
compressedSize = ZSTD_compress(compressed, compressedCapacity,
source, sourceSize, 3);
if (checkError(compressedSize, "ZSTD_compress")) goto cleanup;
restoredSize = ZSTD_decompress(restored, sourceSize, compressed, compressedSize);
if (checkError(restoredSize, "ZSTD_decompress")) goto cleanup;
if (restoredSize != sourceSize || memcmp(source, restored, sourceSize) != 0) {
fputs("round trip mismatch\n", stderr);
goto cleanup;
}
/* This must pull the Rust DDict object and resolve its C helper cycle. */
ddict = ZSTD_createDDict(NULL, 0);
if (ddict == NULL || ZSTD_freeDDict(ddict) != 0) {
fputs("DDict lifecycle failed\n", stderr);
goto cleanup;
}
free(restored);
free(compressed);
return 0;
cleanup:
free(restored);
free(compressed);
return 1;
}