build(meson): link migrated Rust code into native libraries

Build a configuration-matched Cargo archive from Meson and flatten its object
members into static libzstd.  This gives C consumers one archive even though
the migrated Rust objects and remaining C code refer to each other.

Shared libraries whole-archive the Cargo output to retain Rust-only ABI
exports.  The implementation supports Meson 0.50's generated-archive linking
rules, matching HUF mode and 32-bit configuration, and documents cross-build
target and archiver selection.

Test Plan:
- fresh Meson both-build smoke consumers and invalidDictionaries
- modern static/shared/both, forced-HUF, i686, install, and fuzzer paths
- real Meson 0.50 static and shared smoke builds
- cargo clippy, cargo clippy --benches, cargo clippy --tests, and nightly fmt

Refs: build/meson/README.md archive and cross-build documentation

build/meson/README.md

@@ -12,6 +12,12 @@ by Dima Krasner \<dima@dimakrasner.com\>.
 It outputs one `libzstd`, either shared or static, depending on
 `default_library` option.
 
+The migrated compatibility modules are built with Cargo as part of every
+Meson library build.  Static builds flatten the Cargo object members into
+`libzstd.a`, so external C consumers link one archive.  Shared builds use a
+whole-archive link for the Cargo archive, retaining Rust-only public ABI
+exports.
+
 ## How to build
 
 `cd` to this meson directory (`build/meson`)
@@ -36,3 +42,14 @@ meson configure
 ```
 
 See [man meson(1)](https://manpages.debian.org/testing/meson/meson.1.en.html).
+
+For a cross build, set the Cargo target explicitly, for example:
+
+```sh
+meson setup -Dzstd_rust_target=aarch64-unknown-linux-gnu builddir
+```
+
+On 32-bit Linux, the `i686-unknown-linux-gnu` target is selected
+automatically.  Set `-Dzstd_rust_archiver=...` when the C cross toolchain needs
+a non-default archiver.  The `huf_force_decompress_x1` and
+`huf_force_decompress_x2` options select matching C and Rust decoder modes.

build/meson/lib/cargo_staticlib.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+
+"""Build Cargo's staticlib and publish it at a Meson custom-target output."""
+
+from __future__ import print_function
+
+import argparse
+import filecmp
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+
+
+def copy_if_changed(source, destination):
+    if destination.exists() and filecmp.cmp(str(source), str(destination), shallow=False):
+        return
+
+    temporary = destination.with_name(destination.name + '.tmp')
+    shutil.copyfile(str(source), str(temporary))
+    os.replace(str(temporary), str(destination))
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--cargo', required=True)
+    parser.add_argument('--manifest-path', required=True)
+    parser.add_argument('--target-dir', required=True)
+    parser.add_argument('--staticlib-name', required=True)
+    parser.add_argument('--output', required=True)
+    parser.add_argument('--features', default='')
+    parser.add_argument('--target', default='')
+    args = parser.parse_args()
+
+    command = [
+        args.cargo,
+        'build',
+        '--manifest-path',
+        args.manifest_path,
+        '--release',
+        '--target-dir',
+        args.target_dir,
+        '--no-default-features',
+    ]
+    if args.features:
+        command.extend(['--features', args.features])
+    if args.target:
+        command.extend(['--target', args.target])
+    subprocess.check_call(command)
+
+    artifact = Path(args.target_dir)
+    if args.target:
+        artifact /= args.target
+    artifact /= 'release'
+    artifact /= args.staticlib_name
+    if not artifact.is_file():
+        print('Cargo did not produce {}'.format(artifact), file=sys.stderr)
+        return 1
+
+    output = Path(args.output)
+    output.parent.mkdir(parents=True, exist_ok=True)
+    copy_if_changed(artifact, output)
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

build/meson/lib/merge_rust_archive.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+# ################################################################
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+
+"""Flatten Cargo's staticlib into Meson's libzstd static archive."""
+
+from __future__ import print_function
+
+import argparse
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import tempfile
+
+
+def run(command, **kwargs):
+    try:
+        return subprocess.run(command, check=True, **kwargs)
+    except subprocess.CalledProcessError as error:
+        print('command failed: {}'.format(' '.join(command)), file=sys.stderr)
+        return error
+
+
+def extract_members(archiver, archive, destination, prefix):
+    members = subprocess.check_output(
+        [archiver, 't', str(archive)], universal_newlines=True
+    ).splitlines()
+    if not members:
+        raise RuntimeError('{} contains no object members'.format(archive))
+
+    objects = []
+    for index, member in enumerate(members):
+        object_file = destination / '{}-{:04d}.o'.format(prefix, index)
+        with object_file.open('wb') as output:
+            run_result = run(
+                [archiver, 'p', str(archive), member], stdout=output
+            )
+        if isinstance(run_result, subprocess.CalledProcessError):
+            raise RuntimeError('could not extract {} from {}'.format(member, archive))
+        objects.append(object_file)
+    return objects
+
+
+def merge_with_ar(args, output):
+    with tempfile.TemporaryDirectory(prefix='zstd-rust-archive-') as temporary:
+        temporary_path = Path(temporary)
+        c_objects = extract_members(
+            args.archiver, Path(args.c_archive), temporary_path, 'c'
+        )
+        rust_objects = extract_members(
+            args.archiver, Path(args.rust_archive), temporary_path, 'rust'
+        )
+        temporary_output = temporary_path / output.name
+        run_result = run(
+            [args.archiver, 'rcs', str(temporary_output)]
+            + [str(object_file) for object_file in c_objects + rust_objects]
+        )
+        if isinstance(run_result, subprocess.CalledProcessError):
+            raise RuntimeError('could not create {}'.format(output))
+        shutil.move(str(temporary_output), str(output))
+
+
+def merge_with_msvc(args, output):
+    temporary_output = output.with_name(output.name + '.tmp')
+    run_result = run(
+        [
+            args.archiver,
+            '/OUT:{}'.format(temporary_output),
+            args.c_archive,
+            args.rust_archive,
+        ]
+    )
+    if isinstance(run_result, subprocess.CalledProcessError):
+        raise RuntimeError('could not create {}'.format(output))
+    os.replace(str(temporary_output), str(output))
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--archiver', required=True)
+    parser.add_argument('--c-archive', required=True)
+    parser.add_argument('--rust-archive', required=True)
+    parser.add_argument('--output', required=True)
+    parser.add_argument('--msvc', action='store_true')
+    args = parser.parse_args()
+
+    output = Path(args.output)
+    output.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        if args.msvc:
+            merge_with_msvc(args, output)
+        else:
+            merge_with_ar(args, output)
+    except (OSError, RuntimeError, subprocess.CalledProcessError) as error:
+        print('could not merge Rust archive: {}'.format(error), file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

build/meson/lib/meson.build

@@ -46,10 +46,99 @@ libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'),
   join_paths(zstd_rootdir, 'lib/dictBuilder/divsufsort.c'),
   join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.c')]
 
+# C compatibility shims in the source list above are implemented by the Rust
+# static library.  Build a Cargo archive in the Meson build directory so its
+# configuration cannot leak into another Meson setup.
+rust_huf_force_x1 = get_option('huf_force_decompress_x1')
+rust_huf_force_x2 = get_option('huf_force_decompress_x2')
+rust_uses_m32 = false
+foreach c_arg : get_option('c_args')
+  if c_arg == '-m32'
+    rust_uses_m32 = true
+  endif
+  if c_arg == '-DHUF_FORCE_DECOMPRESS_X1' or \
+      c_arg.startswith('-DHUF_FORCE_DECOMPRESS_X1=') or \
+      c_arg == '/DHUF_FORCE_DECOMPRESS_X1' or \
+      c_arg.startswith('/DHUF_FORCE_DECOMPRESS_X1=')
+    rust_huf_force_x1 = true
+  endif
+  if c_arg == '-DHUF_FORCE_DECOMPRESS_X2' or \
+      c_arg.startswith('-DHUF_FORCE_DECOMPRESS_X2=') or \
+      c_arg == '/DHUF_FORCE_DECOMPRESS_X2' or \
+      c_arg.startswith('/DHUF_FORCE_DECOMPRESS_X2=')
+    rust_huf_force_x2 = true
+  endif
+endforeach
+if rust_huf_force_x1 and rust_huf_force_x2
+  error('HUF_FORCE_DECOMPRESS_X1 and HUF_FORCE_DECOMPRESS_X2 are mutually exclusive')
+endif
+
+rust_features = ['compression', 'decompression']
+rust_huf_mode = 'default'
+rust_huf_c_args = []
+if rust_huf_force_x1
+  rust_features += 'huf-force-decompress-x1'
+  rust_huf_mode = 'huf-force-decompress-x1'
+  rust_huf_c_args += '-DHUF_FORCE_DECOMPRESS_X1'
+elif rust_huf_force_x2
+  rust_features += 'huf-force-decompress-x2'
+  rust_huf_mode = 'huf-force-decompress-x2'
+  rust_huf_c_args += '-DHUF_FORCE_DECOMPRESS_X2'
+endif
+
+rust_target = get_option('zstd_rust_target')
+if rust_target == ''
+  if host_machine_os == os_linux and \
+      (host_machine.cpu_family() == 'x86' or rust_uses_m32)
+    rust_target = 'i686-unknown-linux-gnu'
+  elif meson.is_cross_build()
+    error('Set -Dzstd_rust_target to the Cargo target triple for this cross build')
+  endif
+endif
+
+rust_build_config = 'c1-d1-' + rust_huf_mode
+rust_target_dir = join_paths(meson.current_build_dir(), 'rust-target', rust_build_config)
+is_msvc = cc_id == compiler_msvc or cc_id == 'clang-cl'
+rust_staticlib_name = is_msvc ? 'zstd_rs.lib' : 'libzstd_rs.a'
+zstd_staticlib_name = is_msvc ? 'zstd.lib' : 'libzstd.a'
+
+cargo = find_program('cargo', required: true)
+python = find_program('python3', 'python', required: true)
+rust_manifest = files(join_paths(zstd_rootdir, 'rust/Cargo.toml'))
+rust_lockfile = files(join_paths(zstd_rootdir, 'rust/Cargo.lock'))
+cargo_staticlib_py = files('cargo_staticlib.py')
+rust_archive_command = [
+  python,
+  cargo_staticlib_py,
+  '--cargo', cargo,
+  '--manifest-path', rust_manifest,
+  '--target-dir', rust_target_dir,
+  '--staticlib-name', rust_staticlib_name,
+  '--output', '@OUTPUT@',
+  '--features', ','.join(rust_features),
+]
+if rust_target != ''
+  rust_archive_command += ['--target', rust_target]
+endif
+rust_archive = custom_target('zstd_rust_archive',
+  output: rust_staticlib_name,
+  command: rust_archive_command,
+  depend_files: [rust_lockfile],
+  build_always_stale: true)
+
+# Meson's c_args controls compilation only.  A native -m32 configuration also
+# needs its final C and C++ link commands to select the i686 linker emulation.
+if rust_uses_m32
+  m32_link_args = cc.get_supported_link_arguments('-m32')
+  add_project_link_arguments(m32_link_args, language: ['c', 'cpp'])
+endif
+
 # really we need anything that defines __GNUC__ as that is what ZSTD_ASM_SUPPORTED is gated on
-# but these are the two compilers that are supported in tree and actually handle this correctly
+# but these are the two compilers that are supported in tree and actually handle this correctly.
-# Otherwise, explicitly disable assembly.
+# The assembly source is AMD64-only: do not add it for an i686 (-m32) or other
-if [compiler_gcc, compiler_clang].contains(cc_id)
+# cross target, even when the build host itself is x86_64.
+if [compiler_gcc, compiler_clang].contains(cc_id) and \
+    ['x86_64', 'amd64'].contains(host_machine.cpu_family()) and not rust_uses_m32
   libzstd_sources += join_paths(zstd_rootdir, 'lib/decompress/huf_decompress_amd64.S')
 else
   add_project_arguments('-DZSTD_DISABLE_ASM', language: 'c')
@@ -101,6 +190,7 @@ if host_machine_os == os_windows and cc_id == compiler_gcc
   mingw_ansi_stdio_flags = [ '-D__USE_MINGW_ANSI_STDIO' ]
 endif
 libzstd_c_args += mingw_ansi_stdio_flags
+libzstd_c_args += rust_huf_c_args
 
 libzstd_debug_cflags = []
 if use_debug
@@ -115,56 +205,103 @@ if use_debug
 endif
 libzstd_c_args += cc.get_supported_arguments(libzstd_debug_cflags)
 
-libzstd = library('zstd',
+# Keep the C archive separate until Cargo has finished.  The final public
+# static archive below contains the object members of both archives, which is
+# necessary because C shims call Rust and Rust calls C helpers.
+build_static = default_library_type != 'shared'
+libzstd_c_static = static_library('zstd_c',
   libzstd_sources,
   include_directories: libzstd_includes,
   c_args: libzstd_c_args,
   gnu_symbol_visibility: 'hidden',
   dependencies: libzstd_deps,
-  install: true,
+  build_by_default: build_static)
-  version: zstd_libversion)
 
-libzstd_dep = declare_dependency(link_with: libzstd,
+rust_archiver_name = get_option('zstd_rust_archiver')
-  include_directories: join_paths(zstd_rootdir,'lib')) # Do not expose private headers
+if rust_archiver_name == ''
+  rust_archiver_name = is_msvc ? 'lib' : 'ar'
+endif
+rust_archiver = find_program(rust_archiver_name, required: true)
+merge_rust_archive_py = files('merge_rust_archive.py')
+merge_rust_archive_command = [
+  python,
+  merge_rust_archive_py,
+  '--archiver', rust_archiver,
+  '--c-archive', '@INPUT0@',
+  '--rust-archive', '@INPUT1@',
+  '--output', '@OUTPUT@',
+]
+if is_msvc
+  merge_rust_archive_command += '--msvc'
+endif
+libzstd_static = custom_target('zstd_static',
+  input: [libzstd_c_static, rust_archive],
+  output: zstd_staticlib_name,
+  command: merge_rust_archive_command,
+  install: build_static,
+  install_dir: get_option('libdir'),
+  build_by_default: build_static)
+libzstd_static_dep = declare_dependency(
+  sources: libzstd_static,
+  dependencies: libzstd_deps,
+  include_directories: join_paths(zstd_rootdir, 'lib'))
 
-# we link to both:
+# We link to both a shared library (for public symbols) and the C-only static
-# - the shared library (for public symbols)
+# archive (for private symbols).  The latter must stay C-only here: the Rust
-# - the static library (for private symbols)
+# symbols are already supplied by the shared library.  MSVC rejects this
-#
+# combination, so its internal users link the flattened static archive.
-# this is needed because internally private symbols are used all the time, and
+if default_library_type == 'static'
-# -fvisibility=hidden means those cannot be found
+  libzstd_dep = libzstd_static_dep
-if get_option('default_library') == 'static'
+  libzstd_internal_dep = declare_dependency(
-  libzstd_static = libzstd
+    sources: libzstd_static,
-  libzstd_internal_dep = declare_dependency(link_with: libzstd,
+    dependencies: libzstd_deps,
-      include_directories: libzstd_includes)
+    include_directories: libzstd_includes)
 else
-  if get_option('default_library') == 'shared'
+  if host_machine_os == os_darwin
-    libzstd_static = static_library('zstd_objlib',
+    rust_shared_link_args = ['-Wl,-force_load,' + rust_archive.full_path()]
-      objects: libzstd.extract_all_objects(recursive: true),
+  elif is_msvc
-      build_by_default: false)
+    rust_shared_link_args = ['/WHOLEARCHIVE:' + rust_archive.full_path()]
   else
-    libzstd_static = libzstd.get_static_lib()
+    rust_shared_link_args = [
+      '-Wl,--whole-archive',
+      rust_archive.full_path(),
+      '-Wl,--no-whole-archive',
+    ]
   endif
+  libzstd_shared = shared_library('zstd',
+    libzstd_sources,
+    include_directories: libzstd_includes,
+    c_args: libzstd_c_args,
+    gnu_symbol_visibility: 'hidden',
+    dependencies: libzstd_deps,
+    link_args: rust_shared_link_args,
+    link_depends: rust_archive,
+    install: true,
+    version: zstd_libversion)
+  libzstd_dep = declare_dependency(link_with: libzstd_shared,
+    include_directories: join_paths(zstd_rootdir, 'lib')) # Do not expose private headers
 
-  if cc_id == compiler_msvc
+  if is_msvc
-    # msvc does not actually support linking to both, but errors out with:
+    libzstd_internal_dep = declare_dependency(
-    #   error LNK2005: ZSTD_<foo> already defined in zstd.lib(zstd-1.dll)
+      sources: libzstd_static,
-    libzstd_internal_dep = declare_dependency(link_with: libzstd_static,
+      dependencies: libzstd_deps,
       include_directories: libzstd_includes)
   else
-    libzstd_internal_dep = declare_dependency(link_with: libzstd,
+    libzstd_internal_dep = declare_dependency(link_with: libzstd_shared,
       # the static library must be linked after the shared one
-      dependencies: declare_dependency(link_with: libzstd_static),
+      dependencies: declare_dependency(link_with: libzstd_c_static),
       include_directories: libzstd_includes)
   endif
 endif
 
-pkgconfig.generate(libzstd,
+pkgconfig.generate(
   name: 'libzstd',
   filebase: 'libzstd',
   description: 'fast lossless compression algorithm library',
   version: zstd_libversion,
-  url: 'https://facebook.github.io/zstd/')
+  url: 'https://facebook.github.io/zstd/',
+  libraries: ['-L${libdir}', '-lzstd'],
+  libraries_private: libzstd_deps)
 
 install_headers(join_paths(zstd_rootdir, 'lib/zstd.h'),
   join_paths(zstd_rootdir, 'lib/zdict.h'),

build/meson/meson_options.txt

@@ -34,3 +34,16 @@ option('lzma', type: 'feature', value: 'auto',
   description: 'Enable lzma support')
 option('lz4', type: 'feature', value: 'auto',
   description: 'Enable lz4 support')
+
+# The migrated compatibility modules are compiled by Cargo.  A cross build
+# needs the corresponding Rust target triple; leave it empty for native
+# builds (or 32-bit Linux, which is detected automatically).
+option('zstd_rust_target', type: 'string', value: '',
+  description: 'Cargo target triple for the Rust compatibility archive')
+option('zstd_rust_archiver', type: 'string', value: '',
+  description: 'Archiver used to merge Rust objects into libzstd (default: ar)')
+
+option('huf_force_decompress_x1', type: 'boolean', value: false,
+  description: 'Force the Huffman X1 decoder in C and Rust')
+option('huf_force_decompress_x2', type: 'boolean', value: false,
+  description: 'Force the Huffman X2 decoder in C and Rust')

build/meson/tests/meson.build

@@ -35,13 +35,12 @@ testcommon_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
   join_paths(zstd_rootdir, 'programs/benchfn.c'),
   join_paths(zstd_rootdir, 'programs/benchzstd.c')]
 
-testcommon = static_library('testcommon',
+# The final executable carries libzstd: static archives cannot consume a
-  testcommon_sources,
+# custom-generated archive directly on Meson 0.50.
-  # needed due to use of private symbol + -fvisibility=hidden
+testcommon = static_library('testcommon', testcommon_sources)
-  link_with: libzstd_static)
 
 testcommon_dep = declare_dependency(link_with: testcommon,
-  dependencies: libzstd_deps,
+  dependencies: [libzstd_deps, libzstd_static_dep],
   include_directories: libzstd_includes)
 
 datagen_sources = [join_paths(zstd_rootdir, 'tests/datagencli.c'),
@@ -102,6 +101,22 @@ invalidDictionaries = executable('invalidDictionaries',
   dependencies: [ libzstd_dep ],
   install: false)
 
+# A native consumer must need only libzstd.a.  This exercises the flattened
+# C/Rust archive rather than relying on Cargo's archive order at link time.
+rustLibSmokeStatic = executable('rustLibSmokeStatic',
+  join_paths(zstd_rootdir, 'tests/rustLibSmoke.c'),
+  dependencies: [libzstd_static_dep],
+  install: false)
+test('test-rustLibSmokeStatic', rustLibSmokeStatic)
+
+if default_library_type != 'static'
+  rustLibSmokeShared = executable('rustLibSmokeShared',
+    join_paths(zstd_rootdir, 'tests/rustLibSmoke.c'),
+    dependencies: [libzstd_dep],
+    install: false)
+  test('test-rustLibSmokeShared', rustLibSmokeShared)
+endif
+
 if 0 < legacy_level and legacy_level <= 4
   legacy_sources = [join_paths(zstd_rootdir, 'tests/legacy.c')]
   legacy = executable('legacy',