diff --git a/build/meson/README.md b/build/meson/README.md index d393a063f..860be3371 100644 --- a/build/meson/README.md +++ b/build/meson/README.md @@ -12,6 +12,12 @@ by Dima Krasner \. It outputs one `libzstd`, either shared or static, depending on `default_library` option. +The migrated compatibility modules are built with Cargo as part of every +Meson library build. Static builds flatten the Cargo object members into +`libzstd.a`, so external C consumers link one archive. Shared builds use a +whole-archive link for the Cargo archive, retaining Rust-only public ABI +exports. + ## How to build `cd` to this meson directory (`build/meson`) @@ -36,3 +42,14 @@ meson configure ``` See [man meson(1)](https://manpages.debian.org/testing/meson/meson.1.en.html). + +For a cross build, set the Cargo target explicitly, for example: + +```sh +meson setup -Dzstd_rust_target=aarch64-unknown-linux-gnu builddir +``` + +On 32-bit Linux, the `i686-unknown-linux-gnu` target is selected +automatically. Set `-Dzstd_rust_archiver=...` when the C cross toolchain needs +a non-default archiver. The `huf_force_decompress_x1` and +`huf_force_decompress_x2` options select matching C and Rust decoder modes. diff --git a/build/meson/lib/cargo_staticlib.py b/build/meson/lib/cargo_staticlib.py new file mode 100644 index 000000000..2478a171d --- /dev/null +++ b/build/meson/lib/cargo_staticlib.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# ################################################################ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. +# ################################################################ + +"""Build Cargo's staticlib and publish it at a Meson custom-target output.""" + +from __future__ import print_function + +import argparse +import filecmp +import os +from pathlib import Path +import shutil +import subprocess +import sys + + +def copy_if_changed(source, destination): + if destination.exists() and filecmp.cmp(str(source), str(destination), shallow=False): + return + + temporary = destination.with_name(destination.name + '.tmp') + shutil.copyfile(str(source), str(temporary)) + os.replace(str(temporary), str(destination)) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--cargo', required=True) + parser.add_argument('--manifest-path', required=True) + parser.add_argument('--target-dir', required=True) + parser.add_argument('--staticlib-name', required=True) + parser.add_argument('--output', required=True) + parser.add_argument('--features', default='') + parser.add_argument('--target', default='') + args = parser.parse_args() + + command = [ + args.cargo, + 'build', + '--manifest-path', + args.manifest_path, + '--release', + '--target-dir', + args.target_dir, + '--no-default-features', + ] + if args.features: + command.extend(['--features', args.features]) + if args.target: + command.extend(['--target', args.target]) + subprocess.check_call(command) + + artifact = Path(args.target_dir) + if args.target: + artifact /= args.target + artifact /= 'release' + artifact /= args.staticlib_name + if not artifact.is_file(): + print('Cargo did not produce {}'.format(artifact), file=sys.stderr) + return 1 + + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + copy_if_changed(artifact, output) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/build/meson/lib/merge_rust_archive.py b/build/meson/lib/merge_rust_archive.py new file mode 100644 index 000000000..39d601b29 --- /dev/null +++ b/build/meson/lib/merge_rust_archive.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# ################################################################ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. +# ################################################################ + +"""Flatten Cargo's staticlib into Meson's libzstd static archive.""" + +from __future__ import print_function + +import argparse +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + + +def run(command, **kwargs): + try: + return subprocess.run(command, check=True, **kwargs) + except subprocess.CalledProcessError as error: + print('command failed: {}'.format(' '.join(command)), file=sys.stderr) + return error + + +def extract_members(archiver, archive, destination, prefix): + members = subprocess.check_output( + [archiver, 't', str(archive)], universal_newlines=True + ).splitlines() + if not members: + raise RuntimeError('{} contains no object members'.format(archive)) + + objects = [] + for index, member in enumerate(members): + object_file = destination / '{}-{:04d}.o'.format(prefix, index) + with object_file.open('wb') as output: + run_result = run( + [archiver, 'p', str(archive), member], stdout=output + ) + if isinstance(run_result, subprocess.CalledProcessError): + raise RuntimeError('could not extract {} from {}'.format(member, archive)) + objects.append(object_file) + return objects + + +def merge_with_ar(args, output): + with tempfile.TemporaryDirectory(prefix='zstd-rust-archive-') as temporary: + temporary_path = Path(temporary) + c_objects = extract_members( + args.archiver, Path(args.c_archive), temporary_path, 'c' + ) + rust_objects = extract_members( + args.archiver, Path(args.rust_archive), temporary_path, 'rust' + ) + temporary_output = temporary_path / output.name + run_result = run( + [args.archiver, 'rcs', str(temporary_output)] + + [str(object_file) for object_file in c_objects + rust_objects] + ) + if isinstance(run_result, subprocess.CalledProcessError): + raise RuntimeError('could not create {}'.format(output)) + shutil.move(str(temporary_output), str(output)) + + +def merge_with_msvc(args, output): + temporary_output = output.with_name(output.name + '.tmp') + run_result = run( + [ + args.archiver, + '/OUT:{}'.format(temporary_output), + args.c_archive, + args.rust_archive, + ] + ) + if isinstance(run_result, subprocess.CalledProcessError): + raise RuntimeError('could not create {}'.format(output)) + os.replace(str(temporary_output), str(output)) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--archiver', required=True) + parser.add_argument('--c-archive', required=True) + parser.add_argument('--rust-archive', required=True) + parser.add_argument('--output', required=True) + parser.add_argument('--msvc', action='store_true') + args = parser.parse_args() + + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + try: + if args.msvc: + merge_with_msvc(args, output) + else: + merge_with_ar(args, output) + except (OSError, RuntimeError, subprocess.CalledProcessError) as error: + print('could not merge Rust archive: {}'.format(error), file=sys.stderr) + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/build/meson/lib/meson.build b/build/meson/lib/meson.build index d086fc2d7..076e10e74 100644 --- a/build/meson/lib/meson.build +++ b/build/meson/lib/meson.build @@ -46,10 +46,99 @@ libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'), join_paths(zstd_rootdir, 'lib/dictBuilder/divsufsort.c'), join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.c')] +# C compatibility shims in the source list above are implemented by the Rust +# static library. Build a Cargo archive in the Meson build directory so its +# configuration cannot leak into another Meson setup. +rust_huf_force_x1 = get_option('huf_force_decompress_x1') +rust_huf_force_x2 = get_option('huf_force_decompress_x2') +rust_uses_m32 = false +foreach c_arg : get_option('c_args') + if c_arg == '-m32' + rust_uses_m32 = true + endif + if c_arg == '-DHUF_FORCE_DECOMPRESS_X1' or \ + c_arg.startswith('-DHUF_FORCE_DECOMPRESS_X1=') or \ + c_arg == '/DHUF_FORCE_DECOMPRESS_X1' or \ + c_arg.startswith('/DHUF_FORCE_DECOMPRESS_X1=') + rust_huf_force_x1 = true + endif + if c_arg == '-DHUF_FORCE_DECOMPRESS_X2' or \ + c_arg.startswith('-DHUF_FORCE_DECOMPRESS_X2=') or \ + c_arg == '/DHUF_FORCE_DECOMPRESS_X2' or \ + c_arg.startswith('/DHUF_FORCE_DECOMPRESS_X2=') + rust_huf_force_x2 = true + endif +endforeach +if rust_huf_force_x1 and rust_huf_force_x2 + error('HUF_FORCE_DECOMPRESS_X1 and HUF_FORCE_DECOMPRESS_X2 are mutually exclusive') +endif + +rust_features = ['compression', 'decompression'] +rust_huf_mode = 'default' +rust_huf_c_args = [] +if rust_huf_force_x1 + rust_features += 'huf-force-decompress-x1' + rust_huf_mode = 'huf-force-decompress-x1' + rust_huf_c_args += '-DHUF_FORCE_DECOMPRESS_X1' +elif rust_huf_force_x2 + rust_features += 'huf-force-decompress-x2' + rust_huf_mode = 'huf-force-decompress-x2' + rust_huf_c_args += '-DHUF_FORCE_DECOMPRESS_X2' +endif + +rust_target = get_option('zstd_rust_target') +if rust_target == '' + if host_machine_os == os_linux and \ + (host_machine.cpu_family() == 'x86' or rust_uses_m32) + rust_target = 'i686-unknown-linux-gnu' + elif meson.is_cross_build() + error('Set -Dzstd_rust_target to the Cargo target triple for this cross build') + endif +endif + +rust_build_config = 'c1-d1-' + rust_huf_mode +rust_target_dir = join_paths(meson.current_build_dir(), 'rust-target', rust_build_config) +is_msvc = cc_id == compiler_msvc or cc_id == 'clang-cl' +rust_staticlib_name = is_msvc ? 'zstd_rs.lib' : 'libzstd_rs.a' +zstd_staticlib_name = is_msvc ? 'zstd.lib' : 'libzstd.a' + +cargo = find_program('cargo', required: true) +python = find_program('python3', 'python', required: true) +rust_manifest = files(join_paths(zstd_rootdir, 'rust/Cargo.toml')) +rust_lockfile = files(join_paths(zstd_rootdir, 'rust/Cargo.lock')) +cargo_staticlib_py = files('cargo_staticlib.py') +rust_archive_command = [ + python, + cargo_staticlib_py, + '--cargo', cargo, + '--manifest-path', rust_manifest, + '--target-dir', rust_target_dir, + '--staticlib-name', rust_staticlib_name, + '--output', '@OUTPUT@', + '--features', ','.join(rust_features), +] +if rust_target != '' + rust_archive_command += ['--target', rust_target] +endif +rust_archive = custom_target('zstd_rust_archive', + output: rust_staticlib_name, + command: rust_archive_command, + depend_files: [rust_lockfile], + build_always_stale: true) + +# Meson's c_args controls compilation only. A native -m32 configuration also +# needs its final C and C++ link commands to select the i686 linker emulation. +if rust_uses_m32 + m32_link_args = cc.get_supported_link_arguments('-m32') + add_project_link_arguments(m32_link_args, language: ['c', 'cpp']) +endif + # really we need anything that defines __GNUC__ as that is what ZSTD_ASM_SUPPORTED is gated on -# but these are the two compilers that are supported in tree and actually handle this correctly -# Otherwise, explicitly disable assembly. -if [compiler_gcc, compiler_clang].contains(cc_id) +# but these are the two compilers that are supported in tree and actually handle this correctly. +# The assembly source is AMD64-only: do not add it for an i686 (-m32) or other +# cross target, even when the build host itself is x86_64. +if [compiler_gcc, compiler_clang].contains(cc_id) and \ + ['x86_64', 'amd64'].contains(host_machine.cpu_family()) and not rust_uses_m32 libzstd_sources += join_paths(zstd_rootdir, 'lib/decompress/huf_decompress_amd64.S') else add_project_arguments('-DZSTD_DISABLE_ASM', language: 'c') @@ -101,6 +190,7 @@ if host_machine_os == os_windows and cc_id == compiler_gcc mingw_ansi_stdio_flags = [ '-D__USE_MINGW_ANSI_STDIO' ] endif libzstd_c_args += mingw_ansi_stdio_flags +libzstd_c_args += rust_huf_c_args libzstd_debug_cflags = [] if use_debug @@ -115,56 +205,103 @@ if use_debug endif libzstd_c_args += cc.get_supported_arguments(libzstd_debug_cflags) -libzstd = library('zstd', +# Keep the C archive separate until Cargo has finished. The final public +# static archive below contains the object members of both archives, which is +# necessary because C shims call Rust and Rust calls C helpers. +build_static = default_library_type != 'shared' +libzstd_c_static = static_library('zstd_c', libzstd_sources, include_directories: libzstd_includes, c_args: libzstd_c_args, gnu_symbol_visibility: 'hidden', dependencies: libzstd_deps, - install: true, - version: zstd_libversion) + build_by_default: build_static) -libzstd_dep = declare_dependency(link_with: libzstd, - include_directories: join_paths(zstd_rootdir,'lib')) # Do not expose private headers +rust_archiver_name = get_option('zstd_rust_archiver') +if rust_archiver_name == '' + rust_archiver_name = is_msvc ? 'lib' : 'ar' +endif +rust_archiver = find_program(rust_archiver_name, required: true) +merge_rust_archive_py = files('merge_rust_archive.py') +merge_rust_archive_command = [ + python, + merge_rust_archive_py, + '--archiver', rust_archiver, + '--c-archive', '@INPUT0@', + '--rust-archive', '@INPUT1@', + '--output', '@OUTPUT@', +] +if is_msvc + merge_rust_archive_command += '--msvc' +endif +libzstd_static = custom_target('zstd_static', + input: [libzstd_c_static, rust_archive], + output: zstd_staticlib_name, + command: merge_rust_archive_command, + install: build_static, + install_dir: get_option('libdir'), + build_by_default: build_static) +libzstd_static_dep = declare_dependency( + sources: libzstd_static, + dependencies: libzstd_deps, + include_directories: join_paths(zstd_rootdir, 'lib')) -# we link to both: -# - the shared library (for public symbols) -# - the static library (for private symbols) -# -# this is needed because internally private symbols are used all the time, and -# -fvisibility=hidden means those cannot be found -if get_option('default_library') == 'static' - libzstd_static = libzstd - libzstd_internal_dep = declare_dependency(link_with: libzstd, - include_directories: libzstd_includes) +# We link to both a shared library (for public symbols) and the C-only static +# archive (for private symbols). The latter must stay C-only here: the Rust +# symbols are already supplied by the shared library. MSVC rejects this +# combination, so its internal users link the flattened static archive. +if default_library_type == 'static' + libzstd_dep = libzstd_static_dep + libzstd_internal_dep = declare_dependency( + sources: libzstd_static, + dependencies: libzstd_deps, + include_directories: libzstd_includes) else - if get_option('default_library') == 'shared' - libzstd_static = static_library('zstd_objlib', - objects: libzstd.extract_all_objects(recursive: true), - build_by_default: false) + if host_machine_os == os_darwin + rust_shared_link_args = ['-Wl,-force_load,' + rust_archive.full_path()] + elif is_msvc + rust_shared_link_args = ['/WHOLEARCHIVE:' + rust_archive.full_path()] else - libzstd_static = libzstd.get_static_lib() + rust_shared_link_args = [ + '-Wl,--whole-archive', + rust_archive.full_path(), + '-Wl,--no-whole-archive', + ] endif + libzstd_shared = shared_library('zstd', + libzstd_sources, + include_directories: libzstd_includes, + c_args: libzstd_c_args, + gnu_symbol_visibility: 'hidden', + dependencies: libzstd_deps, + link_args: rust_shared_link_args, + link_depends: rust_archive, + install: true, + version: zstd_libversion) + libzstd_dep = declare_dependency(link_with: libzstd_shared, + include_directories: join_paths(zstd_rootdir, 'lib')) # Do not expose private headers - if cc_id == compiler_msvc - # msvc does not actually support linking to both, but errors out with: - # error LNK2005: ZSTD_ already defined in zstd.lib(zstd-1.dll) - libzstd_internal_dep = declare_dependency(link_with: libzstd_static, + if is_msvc + libzstd_internal_dep = declare_dependency( + sources: libzstd_static, + dependencies: libzstd_deps, include_directories: libzstd_includes) else - libzstd_internal_dep = declare_dependency(link_with: libzstd, + libzstd_internal_dep = declare_dependency(link_with: libzstd_shared, # the static library must be linked after the shared one - dependencies: declare_dependency(link_with: libzstd_static), + dependencies: declare_dependency(link_with: libzstd_c_static), include_directories: libzstd_includes) endif endif -pkgconfig.generate(libzstd, +pkgconfig.generate( name: 'libzstd', filebase: 'libzstd', description: 'fast lossless compression algorithm library', version: zstd_libversion, - url: 'https://facebook.github.io/zstd/') + url: 'https://facebook.github.io/zstd/', + libraries: ['-L${libdir}', '-lzstd'], + libraries_private: libzstd_deps) install_headers(join_paths(zstd_rootdir, 'lib/zstd.h'), join_paths(zstd_rootdir, 'lib/zdict.h'), diff --git a/build/meson/meson_options.txt b/build/meson/meson_options.txt index 470517827..09910c55b 100644 --- a/build/meson/meson_options.txt +++ b/build/meson/meson_options.txt @@ -34,3 +34,16 @@ option('lzma', type: 'feature', value: 'auto', description: 'Enable lzma support') option('lz4', type: 'feature', value: 'auto', description: 'Enable lz4 support') + +# The migrated compatibility modules are compiled by Cargo. A cross build +# needs the corresponding Rust target triple; leave it empty for native +# builds (or 32-bit Linux, which is detected automatically). +option('zstd_rust_target', type: 'string', value: '', + description: 'Cargo target triple for the Rust compatibility archive') +option('zstd_rust_archiver', type: 'string', value: '', + description: 'Archiver used to merge Rust objects into libzstd (default: ar)') + +option('huf_force_decompress_x1', type: 'boolean', value: false, + description: 'Force the Huffman X1 decoder in C and Rust') +option('huf_force_decompress_x2', type: 'boolean', value: false, + description: 'Force the Huffman X2 decoder in C and Rust') diff --git a/build/meson/tests/meson.build b/build/meson/tests/meson.build index 71ffc5069..42145093b 100644 --- a/build/meson/tests/meson.build +++ b/build/meson/tests/meson.build @@ -35,13 +35,12 @@ testcommon_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'), join_paths(zstd_rootdir, 'programs/benchfn.c'), join_paths(zstd_rootdir, 'programs/benchzstd.c')] -testcommon = static_library('testcommon', - testcommon_sources, - # needed due to use of private symbol + -fvisibility=hidden - link_with: libzstd_static) +# The final executable carries libzstd: static archives cannot consume a +# custom-generated archive directly on Meson 0.50. +testcommon = static_library('testcommon', testcommon_sources) testcommon_dep = declare_dependency(link_with: testcommon, - dependencies: libzstd_deps, + dependencies: [libzstd_deps, libzstd_static_dep], include_directories: libzstd_includes) datagen_sources = [join_paths(zstd_rootdir, 'tests/datagencli.c'), @@ -102,6 +101,22 @@ invalidDictionaries = executable('invalidDictionaries', dependencies: [ libzstd_dep ], install: false) +# A native consumer must need only libzstd.a. This exercises the flattened +# C/Rust archive rather than relying on Cargo's archive order at link time. +rustLibSmokeStatic = executable('rustLibSmokeStatic', + join_paths(zstd_rootdir, 'tests/rustLibSmoke.c'), + dependencies: [libzstd_static_dep], + install: false) +test('test-rustLibSmokeStatic', rustLibSmokeStatic) + +if default_library_type != 'static' + rustLibSmokeShared = executable('rustLibSmokeShared', + join_paths(zstd_rootdir, 'tests/rustLibSmoke.c'), + dependencies: [libzstd_dep], + install: false) + test('test-rustLibSmokeShared', rustLibSmokeShared) +endif + if 0 < legacy_level and legacy_level <= 4 legacy_sources = [join_paths(zstd_rootdir, 'tests/legacy.c')] legacy = executable('legacy',