ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(fileio): port async pools to Rust

Browse Source 
Replace programs/fileio_asyncio.c with a declaration-only shim and register
Rust's async file-I/O implementation in the library archive. Preserve the C
ABI for pool and job contexts, including DEBUGLEVEL pointer-based pthread
wrappers, and select separate Rust build configurations for those layouts.
Keep ordered-read queue state locked while waiting, drain queued work before
shutdown frees job buffers, and use platform large-file seeks for sparse writes.

Test Plan:
- `cargo test --no-default-features --lib fileio_asyncio` (7 passed).
- `cargo test --no-default-features --features debug-pthread --lib fileio_asyncio` (7 passed).
- `cargo check --all-targets`, clippy library/benches/tests, and nightly fmt check (passed).
- Threaded program build and README round-trip (passed).
- `make -C tests poolTests` and `./tests/poolTests` (passed).
- Current C shim object defines no `AIO_*` symbols; the Rust archive exports all 20 declarations.
- Full standalone all-target Rust tests and the default CLI archive rebuild remain affected by unrelated C/Rust integration and another worker's in-progress benchmark changes.
This commit is contained in:
ddidderr
2026-07-18 01:43:25 +02:00
parent a15bf1d375
commit b6611f9f90
5 changed files with 1773 additions and 656 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/Makefile
+22 -4
View File
@@ -85,7 +85,13 @@ ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
RUST_LEGACY_FEATURES := $(addprefix legacy-v0,$(wordlist $(ZSTD_LEGACY_SUPPORT),7,1 2 3 4 5 6 7))
endif
endif
RUST_BUILD_CONFIG := $(RUST_HUF_MODE)-legacy$(ZSTD_LEGACY_SUPPORT)
RUST_DEBUG_FEATURE :=
RUST_DEBUG_MODE := release
ifneq ($(filter-out 0,$(DEBUGLEVEL)),)
RUST_DEBUG_FEATURE := debug-pthread
RUST_DEBUG_MODE := debug
endif
RUST_BUILD_CONFIG := $(RUST_HUF_MODE)-legacy$(ZSTD_LEGACY_SUPPORT)-$(RUST_DEBUG_MODE)
RUST_TARGET_DIR := $(RUST_DIR)/target/$(RUST_BUILD_CONFIG)
RUST_STATICLIB := $(RUST_TARGET_DIR)/release/libzstd_rs.a
@@ -100,6 +106,9 @@ endif
ifneq ($(RUST_LEGACY_FEATURES),)
RUST_CARGO_FLAGS += --features $(subst $(space),$(comma),$(strip $(RUST_LEGACY_FEATURES)))
endif
ifneq ($(RUST_DEBUG_FEATURE),)
RUST_CARGO_FLAGS += --features $(RUST_DEBUG_FEATURE)
endif
$(RUST_STATICLIB): $(RUST_SOURCES)
$(CARGO) build $(RUST_CARGO_FLAGS)
@@ -121,7 +130,7 @@ $(RUST_CLI_STATICLIB): $(RUST_CLI_SOURCES)
$(RUST_CLI_STATICLIB_32): $(RUST_CLI_SOURCES)
$(CARGO) build $(RUST_CLI_CARGO_FLAGS) --target $(RUST_TARGET_32)
RUST_DECOMPRESS_BUILD_CONFIG := lib-c0-d1-b0-$(RUST_HUF_MODE)
RUST_DECOMPRESS_BUILD_CONFIG := lib-c0-d1-b0-$(RUST_HUF_MODE)-$(RUST_DEBUG_MODE)
RUST_DECOMPRESS_TARGET_DIR := $(RUST_DIR)/target/$(RUST_DECOMPRESS_BUILD_CONFIG)
RUST_DECOMPRESS_STATICLIB := $(RUST_DECOMPRESS_TARGET_DIR)/release/libzstd_rs.a
RUST_DECOMPRESS_CARGO_FLAGS := --manifest-path $(RUST_MANIFEST) --release \
@@ -130,6 +139,9 @@ RUST_DECOMPRESS_CARGO_FLAGS := --manifest-path $(RUST_MANIFEST) --release \
ifneq ($(RUST_HUF_FEATURE),)
RUST_DECOMPRESS_CARGO_FLAGS += --features $(RUST_HUF_FEATURE)
endif
ifneq ($(RUST_DEBUG_FEATURE),)
RUST_DECOMPRESS_CARGO_FLAGS += --features $(RUST_DEBUG_FEATURE)
endif
$(RUST_DECOMPRESS_STATICLIB): $(RUST_SOURCES)
$(CARGO) build $(RUST_DECOMPRESS_CARGO_FLAGS)
@@ -144,23 +156,29 @@ RUST_DECOMPRESS_CLI_CARGO_FLAGS := --manifest-path $(RUST_CLI_MANIFEST) --releas
$(RUST_DECOMPRESS_CLI_STATICLIB): $(RUST_CLI_SOURCES)
$(CARGO) build $(RUST_DECOMPRESS_CLI_CARGO_FLAGS)
RUST_COMPRESS_BUILD_CONFIG := lib-c1-d0-b0-$(RUST_HUF_MODE)
RUST_COMPRESS_BUILD_CONFIG := lib-c1-d0-b0-$(RUST_HUF_MODE)-$(RUST_DEBUG_MODE)
RUST_COMPRESS_TARGET_DIR := $(RUST_DIR)/target/$(RUST_COMPRESS_BUILD_CONFIG)
RUST_COMPRESS_STATICLIB := $(RUST_COMPRESS_TARGET_DIR)/release/libzstd_rs.a
RUST_COMPRESS_CARGO_FLAGS := --manifest-path $(RUST_MANIFEST) --release \
--target-dir $(RUST_COMPRESS_TARGET_DIR) \
--no-default-features --features compression
ifneq ($(RUST_DEBUG_FEATURE),)
RUST_COMPRESS_CARGO_FLAGS += --features $(RUST_DEBUG_FEATURE)
endif
$(RUST_COMPRESS_STATICLIB): $(RUST_SOURCES)
$(CARGO) build $(RUST_COMPRESS_CARGO_FLAGS)
RUST_DICTBUILDER_BUILD_CONFIG := lib-c1-d0-b1-$(RUST_HUF_MODE)
RUST_DICTBUILDER_BUILD_CONFIG := lib-c1-d0-b1-$(RUST_HUF_MODE)-$(RUST_DEBUG_MODE)
RUST_DICTBUILDER_TARGET_DIR := $(RUST_DIR)/target/$(RUST_DICTBUILDER_BUILD_CONFIG)
RUST_DICTBUILDER_STATICLIB := $(RUST_DICTBUILDER_TARGET_DIR)/release/libzstd_rs.a
RUST_DICTBUILDER_CARGO_FLAGS := --manifest-path $(RUST_MANIFEST) --release \
--target-dir $(RUST_DICTBUILDER_TARGET_DIR) \
--no-default-features \
--features compression,dict-builder
ifneq ($(RUST_DEBUG_FEATURE),)
RUST_DICTBUILDER_CARGO_FLAGS += --features $(RUST_DEBUG_FEATURE)
endif
$(RUST_DICTBUILDER_STATICLIB): $(RUST_SOURCES)
$(CARGO) build $(RUST_DICTBUILDER_CARGO_FLAGS)