From 22ec52d5ec25fbd33e7f99e090be2cf66a30d820 Mon Sep 17 00:00:00 2001 From: ddidderr Date: Fri, 10 Jul 2026 21:55:40 +0200 Subject: [PATCH] build(cmake): link migrated Rust code into native libraries Build a configuration-matched Cargo static library during CMake builds and join its object members into the generated static libzstd archive. A single flattened archive avoids C-to-Rust and Rust-to-C archive-order failures for external consumers. Shared builds whole-archive the Cargo output so Rust-only public ABI exports remain visible. The configuration also follows partial compression and decompression builds, forced HUF modes, and supported 32-bit Linux targets. Dedicated CTest smoke consumers cover both linkage forms. Test Plan: - configure and build fresh static and shared CMake smoke targets - ctest -R 'rustLibSmoke(Static|Shared)' --output-on-failure - cargo clippy, cargo clippy --benches, cargo clippy --tests, and nightly fmt - validate partial, forced-HUF, i686, install, and incremental build paths Refs: build/cmake/lib/merge_rust_archive.cmake --- build/cmake/lib/CMakeLists.txt | 177 ++++++++++++++++++++++- build/cmake/lib/merge_rust_archive.cmake | 82 +++++++++++ build/cmake/tests/CMakeLists.txt | 17 +++ 3 files changed, 275 insertions(+), 1 deletion(-) create mode 100644 build/cmake/lib/merge_rust_archive.cmake diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index 4e902a1fb..159cbf063 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -16,6 +16,10 @@ option(ZSTD_BUILD_COMPRESSION "BUILD COMPRESSION MODULE" ON) option(ZSTD_BUILD_DECOMPRESSION "BUILD DECOMPRESSION MODULE" ON) option(ZSTD_BUILD_DICTBUILDER "BUILD DICTBUILDER MODULE" ON) option(ZSTD_BUILD_DEPRECATED "BUILD DEPRECATED MODULE" OFF) +option(ZSTD_HUF_FORCE_DECOMPRESS_X1 "FORCE THE HUF X1 DECODER" OFF) +option(ZSTD_HUF_FORCE_DECOMPRESS_X2 "FORCE THE HUF X2 DECODER" OFF) +set(ZSTD_RUST_TARGET "" CACHE STRING + "Cargo target triple (required when CMake cross-compiles)") set(ZSTDLIB_VISIBLE "" CACHE STRING "Visibility for ZSTDLIB API") set(ZSTDERRORLIB_VISIBLE "" CACHE STRING "Visibility for ZSTDERRORLIB_VISIBLE API") @@ -39,7 +43,8 @@ file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c) if (MSVC) add_compile_options(-DZSTD_DISABLE_ASM) else () - if(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|AMD64.*|x86_64.*|X86_64.*" AND ${ZSTD_HAS_NOEXECSTACK}) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|AMD64.*|x86_64.*|X86_64.*" + AND CMAKE_SIZEOF_VOID_P EQUAL 8 AND ${ZSTD_HAS_NOEXECSTACK}) set(DecompressSources ${DecompressSources} ${LIBRARY_DIR}/decompress/huf_decompress_amd64.S) else() add_compile_options(-DZSTD_DISABLE_ASM) @@ -97,6 +102,135 @@ if (ZSTD_LEGACY_SUPPORT) ${LIBRARY_LEGACY_DIR}/zstd_v07.h) endif () +# The C compatibility shims are backed by a Cargo static library. Keep its +# feature set in lockstep with CMake's partial-library switches, otherwise a +# Rust object can retain references to a C module that was intentionally left +# out of this build. +find_program(ZSTD_CARGO_EXECUTABLE NAMES cargo) +if(NOT ZSTD_CARGO_EXECUTABLE) + message(FATAL_ERROR "Cargo is required to build the migrated Rust modules") +endif() +mark_as_advanced(ZSTD_CARGO_EXECUTABLE ZSTD_RUST_TARGET) + +set(_zstd_huf_c_flags "${CMAKE_C_FLAGS}") +foreach(_zstd_build_type DEBUG RELEASE RELWITHDEBINFO MINSIZEREL) + set(_zstd_c_flag_var "CMAKE_C_FLAGS_${_zstd_build_type}") + set(_zstd_huf_c_flags "${_zstd_huf_c_flags} ${${_zstd_c_flag_var}}") +endforeach() + +set(_zstd_huf_force_x1 ${ZSTD_HUF_FORCE_DECOMPRESS_X1}) +set(_zstd_huf_force_x2 ${ZSTD_HUF_FORCE_DECOMPRESS_X2}) +if(_zstd_huf_c_flags MATCHES "(^|[ \t])(-D|/D)HUF_FORCE_DECOMPRESS_X1(=[^ \t]+)?([ \t]|$)") + set(_zstd_huf_force_x1 ON) +endif() +if(_zstd_huf_c_flags MATCHES "(^|[ \t])(-D|/D)HUF_FORCE_DECOMPRESS_X2(=[^ \t]+)?([ \t]|$)") + set(_zstd_huf_force_x2 ON) +endif() +if(_zstd_huf_force_x1 AND _zstd_huf_force_x2) + message(FATAL_ERROR + "HUF_FORCE_DECOMPRESS_X1 and HUF_FORCE_DECOMPRESS_X2 are mutually exclusive") +endif() + +set(_zstd_rust_c_definitions) +if(ZSTD_HUF_FORCE_DECOMPRESS_X1) + list(APPEND _zstd_rust_c_definitions HUF_FORCE_DECOMPRESS_X1) +endif() +if(ZSTD_HUF_FORCE_DECOMPRESS_X2) + list(APPEND _zstd_rust_c_definitions HUF_FORCE_DECOMPRESS_X2) +endif() + +set(_zstd_rust_features) +set(_zstd_rust_compression 0) +set(_zstd_rust_decompression 0) +if(ZSTD_BUILD_COMPRESSION) + list(APPEND _zstd_rust_features compression) + set(_zstd_rust_compression 1) +endif() +if(ZSTD_BUILD_DECOMPRESSION) + list(APPEND _zstd_rust_features decompression) + set(_zstd_rust_decompression 1) +endif() + +set(_zstd_rust_huf_mode default) +if(_zstd_huf_force_x1) + set(_zstd_rust_huf_mode huf-force-decompress-x1) + if(ZSTD_BUILD_DECOMPRESSION) + list(APPEND _zstd_rust_features huf-force-decompress-x1) + endif() +elseif(_zstd_huf_force_x2) + set(_zstd_rust_huf_mode huf-force-decompress-x2) + if(ZSTD_BUILD_DECOMPRESSION) + list(APPEND _zstd_rust_features huf-force-decompress-x2) + endif() +endif() + +set(_zstd_rust_target "${ZSTD_RUST_TARGET}") +if(NOT _zstd_rust_target AND CMAKE_SYSTEM_NAME STREQUAL "Linux" + AND CMAKE_SIZEOF_VOID_P EQUAL 4) + set(_zstd_rust_x86_32 OFF) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(i[3-6]86|x86)$") + set(_zstd_rust_x86_32 ON) + elseif(CMAKE_C_COMPILER_TARGET MATCHES "^(i[3-6]86|x86)-") + set(_zstd_rust_x86_32 ON) + elseif(_zstd_huf_c_flags MATCHES "(^|[ \t])-m32([ \t]|$)") + set(_zstd_rust_x86_32 ON) + endif() + if(_zstd_rust_x86_32) + set(_zstd_rust_target i686-unknown-linux-gnu) + endif() +endif() +if(NOT _zstd_rust_target AND CMAKE_CROSSCOMPILING) + message(FATAL_ERROR + "Set ZSTD_RUST_TARGET to the Cargo target triple for this cross build") +endif() + +if(_zstd_rust_features) + string(REPLACE ";" "," _zstd_rust_feature_arg "${_zstd_rust_features}") + set(_zstd_rust_feature_args --features "${_zstd_rust_feature_arg}") +endif() + +set(_zstd_rust_build_config + "c${_zstd_rust_compression}-d${_zstd_rust_decompression}-${_zstd_rust_huf_mode}") +set(ZSTD_RUST_MANIFEST "${ZSTD_SOURCE_DIR}/rust/Cargo.toml") +set(ZSTD_RUST_TARGET_DIR + "${CMAKE_CURRENT_BINARY_DIR}/rust-target/${_zstd_rust_build_config}") +set(_zstd_rust_target_args) +set(_zstd_rust_target_component) +if(_zstd_rust_target) + list(APPEND _zstd_rust_target_args --target "${_zstd_rust_target}") + set(_zstd_rust_target_component "/${_zstd_rust_target}") +endif() +if(MSVC) + set(_zstd_rust_staticlib_name zstd_rs.lib) +else() + set(_zstd_rust_staticlib_name libzstd_rs.a) +endif() +set(ZSTD_RUST_STATICLIB + "${ZSTD_RUST_TARGET_DIR}${_zstd_rust_target_component}/release/${_zstd_rust_staticlib_name}") + +# This is deliberately an always-checked target instead of a configure-time + # Rust source glob. The project supports CMake 3.10, predating reliable + # CONFIGURE_DEPENDS handling here, while Cargo already tracks new modules and + # manifest changes precisely. LINK_DEPENDS below still avoids relinking the + # C library when Cargo leaves its archive unchanged. +add_custom_target(zstd_rust_archive + BYPRODUCTS "${ZSTD_RUST_STATICLIB}" + COMMAND "${ZSTD_CARGO_EXECUTABLE}" build + --manifest-path "${ZSTD_RUST_MANIFEST}" + --release + --target-dir "${ZSTD_RUST_TARGET_DIR}" + --no-default-features + ${_zstd_rust_feature_args} + ${_zstd_rust_target_args} + WORKING_DIRECTORY "${ZSTD_SOURCE_DIR}" + COMMENT "Building Rust compatibility archive (${_zstd_rust_build_config})" + VERBATIM) + +function(zstd_attach_rust_archive target) + add_dependencies(${target} zstd_rust_archive) + set_property(TARGET ${target} APPEND PROPERTY LINK_DEPENDS "${ZSTD_RUST_STATICLIB}") +endfunction() + if (MSVC) set(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/libzstd-dll) set(PlatformDependResources ${MSVC_RESOURCE_DIR}/libzstd-dll.rc) @@ -124,6 +258,25 @@ set(library_targets) if (ZSTD_BUILD_SHARED) add_library(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources}) target_include_directories(libzstd_shared INTERFACE $) + if(_zstd_rust_c_definitions) + target_compile_definitions(libzstd_shared PRIVATE ${_zstd_rust_c_definitions}) + endif() + zstd_attach_rust_archive(libzstd_shared) + # Link the complete Cargo archive, rather than only the members reached by + # C code. Several migrated functions are public ABI entry points and are + # otherwise invisible to the shared-library linker's archive scan. + if(APPLE) + target_link_libraries(libzstd_shared PRIVATE + "-Wl,-force_load,${ZSTD_RUST_STATICLIB}") + elseif(MSVC) + target_link_libraries(libzstd_shared PRIVATE + "/WHOLEARCHIVE:${ZSTD_RUST_STATICLIB}") + else() + target_link_libraries(libzstd_shared PRIVATE + "-Wl,--whole-archive" + "${ZSTD_RUST_STATICLIB}" + "-Wl,--no-whole-archive") + endif() list(APPEND library_targets libzstd_shared) if (ZSTD_MULTITHREAD_SUPPORT) set_property(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD") @@ -138,6 +291,28 @@ endif () if (ZSTD_BUILD_STATIC) add_library(libzstd_static STATIC ${Sources} ${Headers}) target_include_directories(libzstd_static INTERFACE $) + if(_zstd_rust_c_definitions) + target_compile_definitions(libzstd_static PRIVATE ${_zstd_rust_c_definitions}) + endif() + zstd_attach_rust_archive(libzstd_static) + if(MSVC) + set(_zstd_archive_style MSVC) + else() + set(_zstd_archive_style AR) + endif() + # CMake builds the C archive first. Merge Cargo's object members after + # that step so external C consumers need only one libzstd archive and do + # not depend on a particular C/Rust archive ordering. + add_custom_command(TARGET libzstd_static POST_BUILD + COMMAND "${CMAKE_COMMAND}" + "-DARCHIVER:FILEPATH=${CMAKE_AR}" + "-DARCHIVER_STYLE:STRING=${_zstd_archive_style}" + "-DRANLIB:FILEPATH=${CMAKE_RANLIB}" + "-DRUST_ARCHIVE:FILEPATH=${ZSTD_RUST_STATICLIB}" + "-DTARGET_ARCHIVE:FILEPATH=$" + "-DWORK_DIR:PATH=${CMAKE_CURRENT_BINARY_DIR}/rust-archive-members" + -P "${CMAKE_CURRENT_LIST_DIR}/merge_rust_archive.cmake" + VERBATIM) list(APPEND library_targets libzstd_static) if (ZSTD_MULTITHREAD_SUPPORT) set_property(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD") diff --git a/build/cmake/lib/merge_rust_archive.cmake b/build/cmake/lib/merge_rust_archive.cmake new file mode 100644 index 000000000..ad0d2dda9 --- /dev/null +++ b/build/cmake/lib/merge_rust_archive.cmake @@ -0,0 +1,82 @@ +# ################################################################ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ################################################################ + +# Merge the Cargo static library into CMake's libzstd static archive. Keeping +# the object members in one archive matters: C objects call migrated Rust +# functions and Rust objects call C helpers, so two separate archives are +# sensitive to the consumer linker's archive scan order. + +foreach(_zstd_required ARCHIVER RUST_ARCHIVE TARGET_ARCHIVE WORK_DIR) + if(NOT DEFINED ${_zstd_required}) + message(FATAL_ERROR "${_zstd_required} is required") + endif() +endforeach() + +file(REMOVE_RECURSE "${WORK_DIR}") +file(MAKE_DIRECTORY "${WORK_DIR}") + +if(ARCHIVER_STYLE STREQUAL "MSVC") + # lib.exe accepts libraries as inputs and writes their object members to a + # new archive. Keep the original C archive separate while it is read, + # because /OUT must not overwrite an input archive. + set(_zstd_c_archive "${WORK_DIR}/libzstd-c.lib") + execute_process( + COMMAND "${CMAKE_COMMAND}" -E copy "${TARGET_ARCHIVE}" "${_zstd_c_archive}" + RESULT_VARIABLE _zstd_copy_result) + if(NOT _zstd_copy_result EQUAL 0) + message(FATAL_ERROR "could not copy ${TARGET_ARCHIVE} for Rust archive merge") + endif() + + execute_process( + COMMAND "${ARCHIVER}" "/OUT:${TARGET_ARCHIVE}" "${_zstd_c_archive}" "${RUST_ARCHIVE}" + RESULT_VARIABLE _zstd_merge_result + OUTPUT_VARIABLE _zstd_merge_output + ERROR_VARIABLE _zstd_merge_error) +else() + execute_process( + COMMAND "${ARCHIVER}" x "${RUST_ARCHIVE}" + WORKING_DIRECTORY "${WORK_DIR}" + RESULT_VARIABLE _zstd_extract_result + OUTPUT_VARIABLE _zstd_extract_output + ERROR_VARIABLE _zstd_extract_error) + if(NOT _zstd_extract_result EQUAL 0) + message(FATAL_ERROR + "could not extract Rust archive ${RUST_ARCHIVE}: ${_zstd_extract_output}${_zstd_extract_error}") + endif() + + file(GLOB _zstd_rust_objects "${WORK_DIR}/*.o" "${WORK_DIR}/*.obj") + if(NOT _zstd_rust_objects) + message(FATAL_ERROR "Rust archive ${RUST_ARCHIVE} did not contain object files") + endif() + + execute_process( + COMMAND "${ARCHIVER}" q "${TARGET_ARCHIVE}" ${_zstd_rust_objects} + RESULT_VARIABLE _zstd_merge_result + OUTPUT_VARIABLE _zstd_merge_output + ERROR_VARIABLE _zstd_merge_error) +endif() + +if(NOT _zstd_merge_result EQUAL 0) + message(FATAL_ERROR + "could not merge Rust archive ${RUST_ARCHIVE}: ${_zstd_merge_output}${_zstd_merge_error}") +endif() + +if(DEFINED RANLIB AND NOT "${RANLIB}" STREQUAL "" AND NOT ARCHIVER_STYLE STREQUAL "MSVC") + execute_process( + COMMAND "${RANLIB}" "${TARGET_ARCHIVE}" + RESULT_VARIABLE _zstd_ranlib_result + OUTPUT_VARIABLE _zstd_ranlib_output + ERROR_VARIABLE _zstd_ranlib_error) + if(NOT _zstd_ranlib_result EQUAL 0) + message(FATAL_ERROR + "could not index ${TARGET_ARCHIVE}: ${_zstd_ranlib_output}${_zstd_ranlib_error}") + endif() +endif() + +file(REMOVE_RECURSE "${WORK_DIR}") diff --git a/build/cmake/tests/CMakeLists.txt b/build/cmake/tests/CMakeLists.txt index 56104a4e3..06eb40d9b 100644 --- a/build/cmake/tests/CMakeLists.txt +++ b/build/cmake/tests/CMakeLists.txt @@ -90,6 +90,23 @@ AddTestFlagsOption(ZSTD_ZSTREAM_FLAGS "$ENV{ZSTREAM_TESTTIME} $ENV{FUZZER_FLAGS} "Semicolon-separated list of flags to pass to the zstreamtest test (see `zstreamtest -h` for usage)") add_test(NAME zstreamtest COMMAND "$" ${ZSTD_ZSTREAM_FLAGS}) +# Verify that a consumer can link the completed static archive alone: migrated +# Rust objects and the C code they call must not remain separate archives. The +# shared variant also checks that whole-archive linking retains Rust-only ABI +# exports such as the DDict lifecycle functions. +add_executable(rustLibSmokeStatic ${TESTS_DIR}/rustLibSmoke.c) +target_link_libraries(rustLibSmokeStatic libzstd_static) +add_test(NAME rustLibSmokeStatic COMMAND "$") +if(ZSTD_BUILD_SHARED) + add_executable(rustLibSmokeShared ${TESTS_DIR}/rustLibSmoke.c) + target_link_libraries(rustLibSmokeShared libzstd_shared) + if(UNIX) + set_property(TARGET rustLibSmokeShared APPEND PROPERTY + BUILD_RPATH "$") + add_test(NAME rustLibSmokeShared COMMAND "$") + endif() +endif() + # # playTests.sh #