From 089f8e5b4ddc675483534bbf4e71e024a09dfe4e Mon Sep 17 00:00:00 2001 From: ddidderr Date: Fri, 10 Jul 2026 20:02:17 +0200 Subject: [PATCH] feat(rust): add common primitive compatibility layer Introduce the Rust static library and move the shared byte, bitstream, CPU, error, xxHash, debug, and public-common implementations into it. Thin C shims preserve the existing header-driven C build while original tests link the Rust archive. This establishes the ABI-safe foundation for later codec and CLI ports; entropy coding, runtime support, codecs, dictionaries, and the CLI remain C. The top-down migration map documents that boundary and its validation path. Test Plan: - cargo fmt --check - cargo test --all-targets - cargo clippy --all-targets -- -D warnings - cargo build --release Refs: rust/README.md --- lib/common/debug.c | 30 +- lib/common/error_private.c | 64 +--- lib/common/xxhash.c | 3 +- lib/common/zstd_common.c | 52 +-- rust/Cargo.lock | 16 + rust/Cargo.toml | 10 + rust/README.md | 57 +++ rust/src/bits.rs | 121 +++++++ rust/src/bitstream.rs | 456 ++++++++++++++++++++++++ rust/src/common.rs | 142 ++++++++ rust/src/cpu.rs | 166 +++++++++ rust/src/debug.rs | 52 +++ rust/src/errors.rs | 149 ++++++++ rust/src/lib.rs | 11 + rust/src/mem.rs | 325 ++++++++++++++++++ rust/src/xxhash.rs | 688 +++++++++++++++++++++++++++++++++++++ rust/src/zstd_common.rs | 63 ++++ tests/Makefile | 1 + 18 files changed, 2270 insertions(+), 136 deletions(-) create mode 100644 rust/Cargo.lock create mode 100644 rust/Cargo.toml create mode 100644 rust/README.md create mode 100644 rust/src/bits.rs create mode 100644 rust/src/bitstream.rs create mode 100644 rust/src/common.rs create mode 100644 rust/src/cpu.rs create mode 100644 rust/src/debug.rs create mode 100644 rust/src/errors.rs create mode 100644 rust/src/lib.rs create mode 100644 rust/src/mem.rs create mode 100644 rust/src/xxhash.rs create mode 100644 rust/src/zstd_common.rs diff --git a/lib/common/debug.c b/lib/common/debug.c index 9d0b7d229..aa2063f32 100644 --- a/lib/common/debug.c +++ b/lib/common/debug.c @@ -1,30 +1,4 @@ -/* ****************************************************************** - * debug - * Part of FSE library - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * You can contact the author at : - * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. -****************************************************************** */ - - -/* - * This module only hosts one global variable - * which can be used to dynamically influence the verbosity of traces, - * such as DEBUGLOG and RAWLOG - */ - #include "debug.h" -#if !defined(ZSTD_LINUX_KERNEL) || (DEBUGLEVEL>=2) -/* We only use this when DEBUGLEVEL>=2, but we get -Werror=pedantic errors if a - * translation unit is empty. So remove this from Linux kernel builds, but - * otherwise just leave it in. - */ -int g_debuglevel = DEBUGLEVEL; -#endif +/* Implementation moved to Rust (rust/src/debug.rs) */ +extern int g_debuglevel; diff --git a/lib/common/error_private.c b/lib/common/error_private.c index 7e7f24f67..ea09f74b0 100644 --- a/lib/common/error_private.c +++ b/lib/common/error_private.c @@ -1,64 +1,4 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -/* The purpose of this file is to have a single list of error strings embedded in binary */ - #include "error_private.h" -const char* ERR_getErrorString(ERR_enum code) -{ -#ifdef ZSTD_STRIP_ERROR_STRINGS - (void)code; - return "Error strings stripped"; -#else - static const char* const notErrorCode = "Unspecified error code"; - switch( code ) - { - case PREFIX(no_error): return "No error detected"; - case PREFIX(GENERIC): return "Error (generic)"; - case PREFIX(prefix_unknown): return "Unknown frame descriptor"; - case PREFIX(version_unsupported): return "Version not supported"; - case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; - case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; - case PREFIX(corruption_detected): return "Data corruption detected"; - case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; - case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification"; - case PREFIX(parameter_unsupported): return "Unsupported parameter"; - case PREFIX(parameter_combination_unsupported): return "Unsupported combination of parameters"; - case PREFIX(parameter_outOfBound): return "Parameter is out of bound"; - case PREFIX(init_missing): return "Context should be init first"; - case PREFIX(memory_allocation): return "Allocation error : not enough memory"; - case PREFIX(workSpace_tooSmall): return "workSpace buffer is not large enough"; - case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; - case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; - case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; - case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; - case PREFIX(cannotProduce_uncompressedBlock): return "This mode cannot generate an uncompressed block"; - case PREFIX(stabilityCondition_notRespected): return "pledged buffer stability condition is not respected"; - case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; - case PREFIX(dictionary_wrong): return "Dictionary mismatch"; - case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples"; - case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; - case PREFIX(srcSize_wrong): return "Src size is incorrect"; - case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer"; - case PREFIX(noForwardProgress_destFull): return "Operation made no progress over multiple calls, due to output buffer being full"; - case PREFIX(noForwardProgress_inputEmpty): return "Operation made no progress over multiple calls, due to input being empty"; - /* following error codes are not stable and may be removed or changed in a future version */ - case PREFIX(frameIndex_tooLarge): return "Frame index is too large"; - case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking"; - case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong"; - case PREFIX(srcBuffer_wrong): return "Source buffer is wrong"; - case PREFIX(sequenceProducer_failed): return "Block-level external sequence producer returned an error code"; - case PREFIX(externalSequences_invalid): return "External sequences are not valid"; - case PREFIX(maxCode): - default: return notErrorCode; - } -#endif -} +// Implementation moved to Rust +extern const char* ERR_getErrorString(ERR_enum code); diff --git a/lib/common/xxhash.c b/lib/common/xxhash.c index 052cd5228..066c2e9c7 100644 --- a/lib/common/xxhash.c +++ b/lib/common/xxhash.c @@ -13,6 +13,7 @@ */ #define XXH_STATIC_LINKING_ONLY /* access advanced declarations */ -#define XXH_IMPLEMENTATION /* access definitions */ #include "xxhash.h" + +/* Implementation moved to rust/src/xxhash.rs. */ diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c index 3f04c22ab..9d234b750 100644 --- a/lib/common/zstd_common.c +++ b/lib/common/zstd_common.c @@ -1,48 +1,10 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - - - -/*-************************************* -* Dependencies -***************************************/ -#define ZSTD_DEPS_NEED_MALLOC #include "error_private.h" #include "zstd_internal.h" - -/*-**************************************** -* Version -******************************************/ -unsigned ZSTD_versionNumber(void) { return ZSTD_VERSION_NUMBER; } - -const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; } - - -/*-**************************************** -* ZSTD Error Management -******************************************/ -#undef ZSTD_isError /* defined within zstd_internal.h */ -/*! ZSTD_isError() : - * tells if a return value is an error code - * symbol is required for external callers */ -unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } - -/*! ZSTD_getErrorName() : - * provides error code string from function result (useful for debugging) */ -const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } - -/*! ZSTD_getError() : - * convert a `size_t` function result into a proper ZSTD_errorCode enum */ -ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } - -/*! ZSTD_getErrorString() : - * provides error code string from enum */ -const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); } +/* Implementation moved to Rust (rust/src/zstd_common.rs) */ +extern unsigned ZSTD_versionNumber(void); +extern const char* ZSTD_versionString(void); +extern unsigned ZSTD_isError(size_t code); +extern const char* ZSTD_getErrorName(size_t code); +extern ZSTD_ErrorCode ZSTD_getErrorCode(size_t code); +extern const char* ZSTD_getErrorString(ZSTD_ErrorCode code); diff --git a/rust/Cargo.lock b/rust/Cargo.lock new file mode 100644 index 000000000..a7c3db532 --- /dev/null +++ b/rust/Cargo.lock @@ -0,0 +1,16 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "zstd-rs" +version = "0.1.0" +dependencies = [ + "libc", +] diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 000000000..591f157a2 --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "zstd-rs" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["staticlib"] + +[dependencies] +libc = "0.2" diff --git a/rust/README.md b/rust/README.md new file mode 100644 index 000000000..ddd2a04d7 --- /dev/null +++ b/rust/README.md @@ -0,0 +1,57 @@ +# Rust rewrite + +This directory contains the in-progress Rust replacement for the zstd library +and command-line program. During the migration, the crate is built as a static +library and linked into the original C test programs. Production C translation +units become declaration-only shims as their implementations move to Rust; the +original C tests remain unchanged and provide compatibility coverage. + +## Component map + +The crate is organized from low-level representation helpers toward the public +zstd ABI: + +- Common primitives + - `mem`, `bits`, `bitstream`, and `cpu` implement byte-order, bitstream, and + target-feature operations used by the codecs. + - `errors`, `debug`, `xxhash`, and `zstd_common` provide common exported ABI + functions and state. + - `common` contains shared frame constants and internal data types. +All entropy coding, compression, decompression, dictionary, legacy, runtime, +and CLI translation units are still C at this initial stage. They must move +before the rewrite is complete. Keeping that boundary explicit prevents a +passing hybrid build from being mistaken for the final all-Rust result. + +## Compatibility boundary + +The public ABI continues to come from the existing headers under `lib/`. +Exported Rust functions therefore use C layout and calling conventions. A C +source file whose implementation has moved to Rust remains in the original +makefile source list as a small shim so header configuration and platform +preprocessor behavior stay available during the transition. + +The original test makefile links `target/release/libzstd_rs.a` with whole-archive +semantics. Rebuild the Rust archive before running C tests so the executable +does not use a stale implementation. + +## Validation + +Run focused Rust checks from this directory: + +```sh +cargo fmt --check +cargo clippy --all-targets -- -D warnings +cargo test --all-targets +cargo build --release +``` + +Then run original compatibility tests from the repository root, starting with +the narrow target for the component being migrated. For example: + +```sh +make -C tests fuzzer +./tests/fuzzer -i1 --no-big-tests +``` + +Broader `tests/Makefile` targets remain the authoritative integration gates as +more of the library and CLI are rewritten. diff --git a/rust/src/bits.rs b/rust/src/bits.rs new file mode 100644 index 000000000..25e927634 --- /dev/null +++ b/rust/src/bits.rs @@ -0,0 +1,121 @@ +#![allow(non_snake_case)] +use crate::mem::*; + +#[inline] +pub fn ZSTD_isPower2(u: usize) -> bool { + (u & (u.wrapping_sub(1))) == 0 +} + +#[inline] +pub fn ZSTD_countTrailingZeros32(val: U32) -> u32 { + assert!(val != 0); + val.trailing_zeros() +} + +#[inline] +pub fn ZSTD_countLeadingZeros32(val: U32) -> u32 { + assert!(val != 0); + val.leading_zeros() +} + +#[inline] +pub fn ZSTD_countTrailingZeros64(val: U64) -> u32 { + assert!(val != 0); + val.trailing_zeros() +} + +#[inline] +pub fn ZSTD_countLeadingZeros64(val: U64) -> u32 { + assert!(val != 0); + val.leading_zeros() +} + +#[inline] +pub fn ZSTD_NbCommonBytes(val: usize) -> u32 { + if MEM_isLittleEndian() { + if MEM_64bits() { + ZSTD_countTrailingZeros64(val as U64) >> 3 + } else { + ZSTD_countTrailingZeros32(val as U32) >> 3 + } + } else { + if MEM_64bits() { + ZSTD_countLeadingZeros64(val as U64) >> 3 + } else { + ZSTD_countLeadingZeros32(val as U32) >> 3 + } + } +} + +#[inline] +pub fn ZSTD_highbit32(val: U32) -> u32 { + assert!(val != 0); + 31 - ZSTD_countLeadingZeros32(val) +} + +#[inline] +pub fn ZSTD_rotateRight_U64(value: U64, count: U32) -> U64 { + assert!(count < 64); + value.rotate_right(count) +} + +#[inline] +pub fn ZSTD_rotateRight_U32(value: U32, count: U32) -> U32 { + assert!(count < 32); + value.rotate_right(count) +} + +#[inline] +pub fn ZSTD_rotateRight_U16(value: U16, count: U32) -> U16 { + assert!(count < 16); + value.rotate_right(count) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn power_of_two_matches_the_c_helper() { + assert!(ZSTD_isPower2(0)); + assert!(ZSTD_isPower2(1)); + assert!(ZSTD_isPower2(1 << 20)); + assert!(!ZSTD_isPower2(3)); + assert!(!ZSTD_isPower2((1 << 20) + 1)); + } + + #[test] + fn bit_counts_cover_both_ends() { + assert_eq!(ZSTD_countTrailingZeros32(1), 0); + assert_eq!(ZSTD_countTrailingZeros32(1 << 31), 31); + assert_eq!(ZSTD_countLeadingZeros32(1), 31); + assert_eq!(ZSTD_countLeadingZeros32(1 << 31), 0); + assert_eq!(ZSTD_countTrailingZeros64(1), 0); + assert_eq!(ZSTD_countTrailingZeros64(1 << 63), 63); + assert_eq!(ZSTD_countLeadingZeros64(1), 63); + assert_eq!(ZSTD_countLeadingZeros64(1 << 63), 0); + assert_eq!(ZSTD_highbit32(1), 0); + assert_eq!(ZSTD_highbit32(1 << 31), 31); + } + + #[test] + fn common_bytes_are_counted_in_memory_order() { + if MEM_isLittleEndian() { + assert_eq!(ZSTD_NbCommonBytes(0x0000_0000_0001_0000), 2); + } else if MEM_64bits() { + assert_eq!(ZSTD_NbCommonBytes(0x0001_0000_0000_0000), 2); + } else { + assert_eq!(ZSTD_NbCommonBytes(0x0001_0000), 2); + } + } + + #[test] + fn rotations_match_expected_wraparound() { + assert_eq!(ZSTD_rotateRight_U16(0x1234, 4), 0x4123); + assert_eq!(ZSTD_rotateRight_U32(0x1234_5678, 8), 0x7812_3456); + assert_eq!( + ZSTD_rotateRight_U64(0x0123_4567_89ab_cdef, 16), + 0xcdef_0123_4567_89ab + ); + } +} diff --git a/rust/src/bitstream.rs b/rust/src/bitstream.rs new file mode 100644 index 000000000..c04986085 --- /dev/null +++ b/rust/src/bitstream.rs @@ -0,0 +1,456 @@ +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] + +use crate::bits::ZSTD_highbit32; +use crate::errors::{ZstdErrorCode, ERROR}; +use crate::mem::{MEM_readLEST, MEM_writeLEST}; +use std::os::raw::c_void; + +pub type BitContainerType = usize; + +pub const STREAM_ACCUMULATOR_MIN_32: u32 = 25; +pub const STREAM_ACCUMULATOR_MIN_64: u32 = 57; + +#[repr(C)] +pub struct BIT_CStream_t { + pub bitContainer: BitContainerType, + pub bitPos: u32, + pub startPtr: *mut i8, + pub ptr: *mut i8, + pub endPtr: *mut i8, +} + +#[repr(C)] +pub struct BIT_DStream_t { + pub bitContainer: BitContainerType, + pub bitsConsumed: u32, + pub ptr: *const i8, + pub start: *const i8, + pub limitPtr: *const i8, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum BIT_DStream_status { + Unfinished = 0, + EndOfBuffer = 1, + Completed = 2, + Overflow = 3, +} + +const BIT_MASK: [usize; 32] = [ + 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, + 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, + 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF, 0x3FFFFFFF, 0x7FFFFFFF, +]; + +#[inline] +pub fn BIT_getLowerBits(bitContainer: BitContainerType, nbBits: u32) -> BitContainerType { + assert!(nbBits < 32); + bitContainer & BIT_MASK[nbBits as usize] +} + +pub unsafe fn BIT_initCStream( + bitC: *mut BIT_CStream_t, + startPtr: *mut c_void, + dstCapacity: usize, +) -> usize { + let s = &mut *bitC; + s.bitContainer = 0; + s.bitPos = 0; + s.startPtr = startPtr as *mut i8; + s.ptr = s.startPtr; + if dstCapacity <= std::mem::size_of::() { + s.endPtr = s.startPtr; + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + s.endPtr = s + .startPtr + .add(dstCapacity - std::mem::size_of::()); + 0 +} + +#[inline] +pub unsafe fn BIT_addBits(bitC: *mut BIT_CStream_t, value: BitContainerType, nbBits: u32) { + let s = &mut *bitC; + assert!(nbBits < 32); + assert!(nbBits + s.bitPos < (std::mem::size_of::() * 8) as u32); + s.bitContainer |= BIT_getLowerBits(value, nbBits) << s.bitPos; + s.bitPos += nbBits; +} + +#[inline] +pub unsafe fn BIT_addBitsFast(bitC: *mut BIT_CStream_t, value: BitContainerType, nbBits: u32) { + let s = &mut *bitC; + assert!((value >> nbBits) == 0); + assert!(nbBits + s.bitPos < (std::mem::size_of::() * 8) as u32); + s.bitContainer |= value << s.bitPos; + s.bitPos += nbBits; +} + +#[inline] +pub unsafe fn BIT_flushBitsFast(bitC: *mut BIT_CStream_t) { + let s = &mut *bitC; + let nbBytes = (s.bitPos >> 3) as usize; + assert!(s.bitPos < (std::mem::size_of::() * 8) as u32); + assert!(s.ptr <= s.endPtr); + + MEM_writeLEST(s.ptr as *mut c_void, s.bitContainer); + + s.ptr = s.ptr.add(nbBytes); + s.bitPos &= 7; + s.bitContainer >>= nbBytes * 8; +} + +#[inline] +pub unsafe fn BIT_flushBits(bitC: *mut BIT_CStream_t) { + let s = &mut *bitC; + let nbBytes = (s.bitPos >> 3) as usize; + assert!(s.bitPos < (std::mem::size_of::() * 8) as u32); + assert!(s.ptr <= s.endPtr); + + MEM_writeLEST(s.ptr as *mut c_void, s.bitContainer); + + let next_ptr = s.ptr.add(nbBytes); + if next_ptr > s.endPtr { + s.ptr = s.endPtr; + } else { + s.ptr = next_ptr; + } + s.bitPos &= 7; + s.bitContainer >>= nbBytes * 8; +} + +pub unsafe fn BIT_closeCStream(bitC: *mut BIT_CStream_t) -> usize { + BIT_addBitsFast(bitC, 1, 1); + BIT_flushBits(bitC); + let s = &*bitC; + if s.ptr >= s.endPtr { + return 0; + } + (s.ptr as isize - s.startPtr as isize) as usize + (if s.bitPos > 0 { 1 } else { 0 }) +} + +pub unsafe fn BIT_initDStream( + bitD: *mut BIT_DStream_t, + srcBuffer: *const c_void, + srcSize: usize, +) -> usize { + let s = &mut *bitD; + if srcSize < 1 { + std::ptr::write_bytes(bitD, 0, 1); + return ERROR(ZstdErrorCode::SrcSizeWrong); + } + + s.start = srcBuffer as *const i8; + s.limitPtr = s.start.add(std::mem::size_of::()); + + if srcSize >= std::mem::size_of::() { + s.ptr = s + .start + .add(srcSize) + .offset(-(std::mem::size_of::() as isize)); + s.bitContainer = MEM_readLEST(s.ptr as *const c_void); + let lastByte = *s.start.offset(srcSize as isize - 1) as u8; + s.bitsConsumed = if lastByte != 0 { + 8 - ZSTD_highbit32(lastByte as u32) + } else { + 0 + }; + if lastByte == 0 { + return ERROR(ZstdErrorCode::Generic); + } + } else { + s.ptr = s.start; + s.bitContainer = *(s.start as *const u8) as BitContainerType; + let mut container = s.bitContainer; + let src_bytes = std::slice::from_raw_parts(s.start as *const u8, srcSize); + + for (i, &byte) in src_bytes.iter().enumerate().skip(1) { + container += (byte as BitContainerType) << (i * 8); + } + s.bitContainer = container; + + let lastByte = src_bytes[srcSize - 1]; + s.bitsConsumed = if lastByte != 0 { + 8 - ZSTD_highbit32(lastByte as u32) + } else { + 0 + }; + if lastByte == 0 { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + s.bitsConsumed += (std::mem::size_of::() as u32 - srcSize as u32) * 8; + } + + srcSize +} + +#[inline] +pub fn BIT_getUpperBits(bitContainer: BitContainerType, start: u32) -> BitContainerType { + bitContainer >> start +} + +#[inline] +pub fn BIT_getMiddleBits( + bitContainer: BitContainerType, + start: u32, + nbBits: u32, +) -> BitContainerType { + let regMask = (std::mem::size_of::() * 8 - 1) as u32; + assert!(nbBits < 32); + (bitContainer >> (start & regMask)) & ((1usize << nbBits) - 1) +} + +#[inline] +pub unsafe fn BIT_lookBits(bitD: *const BIT_DStream_t, nbBits: u32) -> BitContainerType { + let s = &*bitD; + let register_bits = (std::mem::size_of::() * 8) as u32; + BIT_getMiddleBits( + s.bitContainer, + register_bits + .wrapping_sub(s.bitsConsumed) + .wrapping_sub(nbBits), + nbBits, + ) +} + +#[inline] +pub unsafe fn BIT_lookBitsFast(bitD: *const BIT_DStream_t, nbBits: u32) -> BitContainerType { + let s = &*bitD; + let regMask = (std::mem::size_of::() * 8 - 1) as u32; + assert!(nbBits >= 1); + (s.bitContainer << (s.bitsConsumed & regMask)) >> ((regMask + 1 - nbBits) & regMask) +} + +#[inline] +pub unsafe fn BIT_skipBits(bitD: *mut BIT_DStream_t, nbBits: u32) { + let s = &mut *bitD; + s.bitsConsumed += nbBits; +} + +#[inline] +pub unsafe fn BIT_readBits(bitD: *mut BIT_DStream_t, nbBits: u32) -> BitContainerType { + let value = BIT_lookBits(bitD, nbBits); + BIT_skipBits(bitD, nbBits); + value +} + +#[inline] +pub unsafe fn BIT_readBitsFast(bitD: *mut BIT_DStream_t, nbBits: u32) -> BitContainerType { + let value = BIT_lookBitsFast(bitD, nbBits); + assert!(nbBits >= 1); + BIT_skipBits(bitD, nbBits); + value +} + +#[inline] +unsafe fn BIT_reloadDStream_internal(bitD: *mut BIT_DStream_t) -> BIT_DStream_status { + let s = &mut *bitD; + assert!(s.bitsConsumed <= (std::mem::size_of::() * 8) as u32); + s.ptr = s.ptr.offset(-(s.bitsConsumed as isize / 8)); + assert!(s.ptr >= s.start); + s.bitsConsumed &= 7; + s.bitContainer = MEM_readLEST(s.ptr as *const c_void); + BIT_DStream_status::Unfinished +} + +#[inline] +pub unsafe fn BIT_reloadDStreamFast(bitD: *mut BIT_DStream_t) -> BIT_DStream_status { + let s = &*bitD; + if s.ptr < s.limitPtr { + return BIT_DStream_status::Overflow; + } + BIT_reloadDStream_internal(bitD) +} + +#[inline] +pub unsafe fn BIT_reloadDStream(bitD: *mut BIT_DStream_t) -> BIT_DStream_status { + let s = &mut *bitD; + if s.bitsConsumed > (std::mem::size_of::() * 8) as u32 { + static ZERO_FILLED: BitContainerType = 0; + s.ptr = &ZERO_FILLED as *const _ as *const i8; + return BIT_DStream_status::Overflow; + } + + assert!(s.ptr >= s.start); + + if s.ptr >= s.limitPtr { + return BIT_reloadDStream_internal(bitD); + } + if s.ptr == s.start { + if s.bitsConsumed < (std::mem::size_of::() * 8) as u32 { + return BIT_DStream_status::EndOfBuffer; + } + return BIT_DStream_status::Completed; + } + + let nbBytes = s.bitsConsumed / 8; + let mut result = BIT_DStream_status::Unfinished; + let available = s.ptr.offset_from(s.start) as u32; + let ptr_offset = if nbBytes > available { + result = BIT_DStream_status::EndOfBuffer; + available + } else { + nbBytes + }; + + s.ptr = s.ptr.sub(ptr_offset as usize); + s.bitsConsumed -= ptr_offset * 8; + s.bitContainer = MEM_readLEST(s.ptr as *const c_void); + result +} + +#[inline] +pub unsafe fn BIT_endOfDStream(DStream: *const BIT_DStream_t) -> u32 { + let s = &*DStream; + if s.ptr == s.start && s.bitsConsumed == (std::mem::size_of::() * 8) as u32 { + 1 + } else { + 0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::mem::{size_of, zeroed}; + + #[test] + fn short_sources_are_loaded_little_endian_and_counted_in_bits() { + let width = size_of::(); + for src_size in 1..width { + let mut source = vec![0u8; src_size]; + for (index, byte) in source.iter_mut().enumerate() { + *byte = (index as u8).wrapping_mul(37).wrapping_add(1); + } + source[src_size - 1] |= 0x80; + + let mut stream: BIT_DStream_t = unsafe { zeroed() }; + let result = unsafe { + BIT_initDStream(&mut stream, source.as_ptr().cast::(), source.len()) + }; + assert_eq!(result, src_size); + + let expected = source + .iter() + .enumerate() + .fold(0usize, |value, (index, byte)| { + value | (usize::from(*byte) << (index * 8)) + }); + assert_eq!(stream.bitContainer, expected); + let last = u32::from(source[src_size - 1]); + let expected_consumed = + (width as u32 - src_size as u32) * 8 + 8 - (31 - last.leading_zeros()); + assert_eq!(stream.bitsConsumed, expected_consumed); + } + } + + #[test] + fn short_stream_round_trip_is_lifo() { + let mut encoded = [0u8; 64]; + let mut encoder: BIT_CStream_t = unsafe { zeroed() }; + assert_eq!( + unsafe { + BIT_initCStream( + &mut encoder, + encoded.as_mut_ptr().cast::(), + encoded.len(), + ) + }, + 0 + ); + unsafe { + BIT_addBits(&mut encoder, 0b101, 3); + BIT_addBits(&mut encoder, 0b1_0010, 5); + BIT_addBits(&mut encoder, 0b10_1010_1010, 10); + } + let encoded_size = unsafe { BIT_closeCStream(&mut encoder) }; + assert_eq!(encoded_size, 3); + + let mut decoder: BIT_DStream_t = unsafe { zeroed() }; + assert_eq!( + unsafe { + BIT_initDStream( + &mut decoder, + encoded.as_ptr().cast::(), + encoded_size, + ) + }, + encoded_size + ); + assert_eq!(unsafe { BIT_readBits(&mut decoder, 10) }, 0b10_1010_1010); + assert_eq!(unsafe { BIT_readBits(&mut decoder, 5) }, 0b1_0010); + assert_eq!(unsafe { BIT_readBits(&mut decoder, 3) }, 0b101); + assert_eq!( + unsafe { BIT_reloadDStream(&mut decoder) }, + BIT_DStream_status::Completed + ); + assert_eq!(unsafe { BIT_endOfDStream(&decoder) }, 1); + } + + #[test] + fn reload_covers_fast_and_cautious_paths() { + let width = size_of::(); + let mut source = vec![0u8; width * 2]; + for (index, byte) in source.iter_mut().enumerate() { + *byte = index as u8 + 1; + } + *source.last_mut().unwrap() |= 0x80; + + let mut stream: BIT_DStream_t = unsafe { zeroed() }; + assert_eq!( + unsafe { BIT_initDStream(&mut stream, source.as_ptr().cast::(), source.len()) }, + source.len() + ); + stream.bitsConsumed = (width * 8) as u32; + assert_eq!( + unsafe { BIT_reloadDStream(&mut stream) }, + BIT_DStream_status::Unfinished + ); + assert_eq!(stream.ptr, stream.start); + assert_eq!(stream.bitsConsumed, 0); + + source.truncate(width + 3); + *source.last_mut().unwrap() |= 0x80; + assert_eq!( + unsafe { BIT_initDStream(&mut stream, source.as_ptr().cast::(), source.len()) }, + source.len() + ); + stream.bitsConsumed = (width * 8) as u32; + assert_eq!( + unsafe { BIT_reloadDStream(&mut stream) }, + BIT_DStream_status::EndOfBuffer + ); + assert_eq!(stream.ptr, stream.start); + assert_eq!(stream.bitsConsumed, (width * 8 - 24) as u32); + } + + #[test] + fn invalid_sizes_and_missing_end_mark_report_c_errors() { + let width = size_of::(); + let mut encoder: BIT_CStream_t = unsafe { zeroed() }; + let mut destination = vec![0u8; width]; + assert_eq!( + unsafe { + BIT_initCStream( + &mut encoder, + destination.as_mut_ptr().cast::(), + destination.len(), + ) + }, + ERROR(ZstdErrorCode::DstSizeTooSmall) + ); + + let mut decoder: BIT_DStream_t = unsafe { zeroed() }; + assert_eq!( + unsafe { BIT_initDStream(&mut decoder, std::ptr::null(), 0) }, + ERROR(ZstdErrorCode::SrcSizeWrong) + ); + let zero = [0u8; 1]; + assert_eq!( + unsafe { BIT_initDStream(&mut decoder, zero.as_ptr().cast(), zero.len()) }, + ERROR(ZstdErrorCode::CorruptionDetected) + ); + } +} diff --git a/rust/src/common.rs b/rust/src/common.rs new file mode 100644 index 000000000..ac8e394bc --- /dev/null +++ b/rust/src/common.rs @@ -0,0 +1,142 @@ +#![allow(non_snake_case)] +use crate::mem::*; + +pub const ZSTD_OPT_NUM: usize = 1 << 12; +pub const ZSTD_REP_NUM: usize = 3; +pub const REP_START_VALUE: [U32; ZSTD_REP_NUM] = [1, 4, 8]; + +pub const ZSTD_WINDOWLOG_ABSOLUTEMIN: usize = 10; +pub const ZSTD_FCS_FIELD_SIZE: [usize; 4] = [0, 2, 4, 8]; +pub const ZSTD_DID_FIELD_SIZE: [usize; 4] = [0, 1, 2, 4]; + +pub const ZSTD_FRAMEIDSIZE: usize = 4; +pub const ZSTD_BLOCKHEADERSIZE: usize = 3; +pub const ZSTD_FRAMECHECKSUMSIZE: usize = 4; + +pub const MIN_SEQUENCES_SIZE: usize = 1; +pub const MIN_CBLOCK_SIZE: usize = 2; +pub const MIN_LITERALS_FOR_4_STREAMS: usize = 6; + +pub const LONGNBSEQ: U32 = 0x7F00; +pub const MINMATCH: usize = 3; + +pub const LITBITS: usize = 8; +pub const LIT_HUF_LOG: usize = 11; +pub const MAX_LIT: usize = (1 << LITBITS) - 1; +pub const MAX_ML: usize = 52; +pub const MAX_LL: usize = 35; +pub const DEFAULT_MAX_OFF: usize = 28; +pub const MAX_OFF: usize = 31; +pub const MAX_SEQ: usize = 52; + +pub const ML_FSE_LOG: usize = 9; +pub const LL_FSE_LOG: usize = 9; +pub const OFF_FSE_LOG: usize = 8; +pub const MAX_FSE_LOG: usize = 9; + +pub const MAX_ML_BITS: usize = 16; +pub const MAX_LL_BITS: usize = 16; + +pub const ZSTD_MAX_HUF_HEADER_SIZE: usize = 128; +pub const ZSTD_MAX_FSE_HEADERS_SIZE: usize = 133; + +pub const LL_BITS: [U8; MAX_LL + 1] = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16, +]; + +pub const LL_DEFAULT_NORM: [S16; MAX_LL + 1] = [ + 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, + -1, -1, -1, -1, +]; + +pub const LL_DEFAULT_NORM_LOG: U32 = 6; + +pub const ML_BITS: [U8; MAX_ML + 1] = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, +]; + +pub const ML_DEFAULT_NORM: [S16; MAX_ML + 1] = [ + 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1, +]; + +pub const ML_DEFAULT_NORM_LOG: U32 = 6; + +pub const OF_DEFAULT_NORM: [S16; DEFAULT_MAX_OFF + 1] = [ + 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, +]; + +pub const OF_DEFAULT_NORM_LOG: U32 = 5; + +#[repr(C)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum BlockType { + Raw, + Rle, + Compressed, + Reserved, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum SymbolEncodingType { + Basic, + Rle, + Compressed, + Repeat, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum BufferMode { + Buffered = 0, + Stable = 1, +} + +#[repr(C)] +pub struct FrameSizeInfo { + pub nb_blocks: usize, + pub compressed_size: usize, + pub decompressed_bound: u64, +} + +#[repr(C)] +pub struct BlockProperties { + pub block_type: BlockType, + pub last_block: U32, + pub orig_size: U32, +} + +#[inline] +pub fn ZSTD_cpuSupportsBmi2() -> bool { + crate::cpu::ZSTD_cpuSupportsBmi2() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::mem::{align_of, size_of}; + + #[test] + fn normalized_distributions_have_the_declared_table_sizes() { + let ll_sum: i32 = LL_DEFAULT_NORM.iter().map(|&v| i32::from(v.abs())).sum(); + let ml_sum: i32 = ML_DEFAULT_NORM.iter().map(|&v| i32::from(v.abs())).sum(); + let of_sum: i32 = OF_DEFAULT_NORM.iter().map(|&v| i32::from(v.abs())).sum(); + assert_eq!(ll_sum, 1 << LL_DEFAULT_NORM_LOG); + assert_eq!(ml_sum, 1 << ML_DEFAULT_NORM_LOG); + assert_eq!(of_sum, 1 << OF_DEFAULT_NORM_LOG); + } + + #[test] + fn c_layout_types_match_the_header() { + assert_eq!(size_of::(), size_of::()); + assert_eq!(size_of::(), size_of::()); + assert_eq!(size_of::(), size_of::()); + assert_eq!(size_of::(), 2 * size_of::() + 8); + assert_eq!(align_of::(), align_of::()); + assert_eq!(size_of::(), 3 * size_of::()); + assert_eq!(align_of::(), align_of::()); + } +} diff --git a/rust/src/cpu.rs b/rust/src/cpu.rs new file mode 100644 index 000000000..5d0f5fee5 --- /dev/null +++ b/rust/src/cpu.rs @@ -0,0 +1,166 @@ +#![allow(non_snake_case)] +use crate::mem::U32; + +#[repr(C)] +#[derive(Debug, Copy, Clone, Default)] +pub struct ZstdCpuid { + pub f1c: U32, + pub f1d: U32, + pub f7b: U32, + pub f7c: U32, +} + +#[inline] +pub fn ZSTD_cpuid() -> ZstdCpuid { + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + cpuid_x86() + } + #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] + { + ZstdCpuid::default() + } +} + +#[cfg(target_arch = "x86")] +use core::arch::x86::{__cpuid, __cpuid_count, CpuidResult}; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::{__cpuid, __cpuid_count, CpuidResult}; + +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +fn cpuid_x86() -> ZstdCpuid { + let mut out = ZstdCpuid::default(); + let max: CpuidResult = __cpuid(0); + if max.eax >= 1 { + let r = __cpuid(1); + out.f1c = r.ecx; + out.f1d = r.edx; + } + if max.eax >= 7 { + let r = __cpuid_count(7, 0); + out.f7b = r.ebx; + out.f7c = r.ecx; + } + out +} + +macro_rules! cpuid_bit { + ($fn_name:ident, $field:ident, $bit:expr) => { + #[inline] + pub fn $fn_name(cpuid: ZstdCpuid) -> bool { + (cpuid.$field & (1u32 << $bit)) != 0 + } + }; +} + +cpuid_bit!(ZSTD_cpuid_sse3, f1c, 0); +cpuid_bit!(ZSTD_cpuid_pclmuldq, f1c, 1); +cpuid_bit!(ZSTD_cpuid_dtes64, f1c, 2); +cpuid_bit!(ZSTD_cpuid_monitor, f1c, 3); +cpuid_bit!(ZSTD_cpuid_dscpl, f1c, 4); +cpuid_bit!(ZSTD_cpuid_vmx, f1c, 5); +cpuid_bit!(ZSTD_cpuid_smx, f1c, 6); +cpuid_bit!(ZSTD_cpuid_eist, f1c, 7); +cpuid_bit!(ZSTD_cpuid_tm2, f1c, 8); +cpuid_bit!(ZSTD_cpuid_ssse3, f1c, 9); +cpuid_bit!(ZSTD_cpuid_cnxtid, f1c, 10); +cpuid_bit!(ZSTD_cpuid_fma, f1c, 12); +cpuid_bit!(ZSTD_cpuid_cx16, f1c, 13); +cpuid_bit!(ZSTD_cpuid_xtpr, f1c, 14); +cpuid_bit!(ZSTD_cpuid_pdcm, f1c, 15); +cpuid_bit!(ZSTD_cpuid_pcid, f1c, 17); +cpuid_bit!(ZSTD_cpuid_dca, f1c, 18); +cpuid_bit!(ZSTD_cpuid_sse41, f1c, 19); +cpuid_bit!(ZSTD_cpuid_sse42, f1c, 20); +cpuid_bit!(ZSTD_cpuid_x2apic, f1c, 21); +cpuid_bit!(ZSTD_cpuid_movbe, f1c, 22); +cpuid_bit!(ZSTD_cpuid_popcnt, f1c, 23); +cpuid_bit!(ZSTD_cpuid_tscdeadline, f1c, 24); +cpuid_bit!(ZSTD_cpuid_aes, f1c, 25); +cpuid_bit!(ZSTD_cpuid_xsave, f1c, 26); +cpuid_bit!(ZSTD_cpuid_osxsave, f1c, 27); +cpuid_bit!(ZSTD_cpuid_avx, f1c, 28); +cpuid_bit!(ZSTD_cpuid_f16c, f1c, 29); +cpuid_bit!(ZSTD_cpuid_rdrand, f1c, 30); +cpuid_bit!(ZSTD_cpuid_fpu, f1d, 0); +cpuid_bit!(ZSTD_cpuid_vme, f1d, 1); +cpuid_bit!(ZSTD_cpuid_de, f1d, 2); +cpuid_bit!(ZSTD_cpuid_pse, f1d, 3); +cpuid_bit!(ZSTD_cpuid_tsc, f1d, 4); +cpuid_bit!(ZSTD_cpuid_msr, f1d, 5); +cpuid_bit!(ZSTD_cpuid_pae, f1d, 6); +cpuid_bit!(ZSTD_cpuid_mce, f1d, 7); +cpuid_bit!(ZSTD_cpuid_cx8, f1d, 8); +cpuid_bit!(ZSTD_cpuid_apic, f1d, 9); +cpuid_bit!(ZSTD_cpuid_sep, f1d, 11); +cpuid_bit!(ZSTD_cpuid_mtrr, f1d, 12); +cpuid_bit!(ZSTD_cpuid_pge, f1d, 13); +cpuid_bit!(ZSTD_cpuid_mca, f1d, 14); +cpuid_bit!(ZSTD_cpuid_cmov, f1d, 15); +cpuid_bit!(ZSTD_cpuid_pat, f1d, 16); +cpuid_bit!(ZSTD_cpuid_pse36, f1d, 17); +cpuid_bit!(ZSTD_cpuid_psn, f1d, 18); +cpuid_bit!(ZSTD_cpuid_clfsh, f1d, 19); +cpuid_bit!(ZSTD_cpuid_ds, f1d, 21); +cpuid_bit!(ZSTD_cpuid_acpi, f1d, 22); +cpuid_bit!(ZSTD_cpuid_mmx, f1d, 23); +cpuid_bit!(ZSTD_cpuid_fxsr, f1d, 24); +cpuid_bit!(ZSTD_cpuid_sse, f1d, 25); +cpuid_bit!(ZSTD_cpuid_sse2, f1d, 26); +cpuid_bit!(ZSTD_cpuid_ss, f1d, 27); +cpuid_bit!(ZSTD_cpuid_htt, f1d, 28); +cpuid_bit!(ZSTD_cpuid_tm, f1d, 29); +cpuid_bit!(ZSTD_cpuid_pbe, f1d, 31); +cpuid_bit!(ZSTD_cpuid_bmi1, f7b, 3); +cpuid_bit!(ZSTD_cpuid_hle, f7b, 4); +cpuid_bit!(ZSTD_cpuid_avx2, f7b, 5); +cpuid_bit!(ZSTD_cpuid_smep, f7b, 7); +cpuid_bit!(ZSTD_cpuid_bmi2, f7b, 8); +cpuid_bit!(ZSTD_cpuid_erms, f7b, 9); +cpuid_bit!(ZSTD_cpuid_invpcid, f7b, 10); +cpuid_bit!(ZSTD_cpuid_rtm, f7b, 11); +cpuid_bit!(ZSTD_cpuid_mpx, f7b, 14); +cpuid_bit!(ZSTD_cpuid_avx512f, f7b, 16); +cpuid_bit!(ZSTD_cpuid_avx512dq, f7b, 17); +cpuid_bit!(ZSTD_cpuid_rdseed, f7b, 18); +cpuid_bit!(ZSTD_cpuid_adx, f7b, 19); +cpuid_bit!(ZSTD_cpuid_smap, f7b, 20); +cpuid_bit!(ZSTD_cpuid_avx512ifma, f7b, 21); +cpuid_bit!(ZSTD_cpuid_pcommit, f7b, 22); +cpuid_bit!(ZSTD_cpuid_clflushopt, f7b, 23); +cpuid_bit!(ZSTD_cpuid_clwb, f7b, 24); +cpuid_bit!(ZSTD_cpuid_avx512pf, f7b, 26); +cpuid_bit!(ZSTD_cpuid_avx512er, f7b, 27); +cpuid_bit!(ZSTD_cpuid_avx512cd, f7b, 28); +cpuid_bit!(ZSTD_cpuid_sha, f7b, 29); +cpuid_bit!(ZSTD_cpuid_avx512bw, f7b, 30); +cpuid_bit!(ZSTD_cpuid_avx512vl, f7b, 31); +cpuid_bit!(ZSTD_cpuid_prefetchwt1, f7c, 0); +cpuid_bit!(ZSTD_cpuid_avx512vbmi, f7c, 1); + +#[inline] +pub fn ZSTD_cpuSupportsBmi2() -> bool { + let cpuid = ZSTD_cpuid(); + ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn feature_helpers_read_the_header_defined_bits() { + let cpuid = ZstdCpuid { + f1c: 1 << 20, + f1d: 1 << 26, + f7b: (1 << 3) | (1 << 8), + f7c: 1 << 1, + }; + assert!(ZSTD_cpuid_sse42(cpuid)); + assert!(ZSTD_cpuid_sse2(cpuid)); + assert!(ZSTD_cpuid_bmi1(cpuid)); + assert!(ZSTD_cpuid_bmi2(cpuid)); + assert!(ZSTD_cpuid_avx512vbmi(cpuid)); + assert!(!ZSTD_cpuid_avx2(cpuid)); + } +} diff --git a/rust/src/debug.rs b/rust/src/debug.rs new file mode 100644 index 000000000..8deb26e7f --- /dev/null +++ b/rust/src/debug.rs @@ -0,0 +1,52 @@ +#![allow(non_snake_case)] + +/// Global debug verbosity level (C `g_debuglevel` from debug.c). +/// Only meaningful when debug logging is enabled at compile time. +#[no_mangle] +pub static mut g_debuglevel: i32 = 0; + +/// Compile-time debug level for this Rust build (matches C `DEBUGLEVEL` default 0). +pub const DEBUGLEVEL: i32 = 0; + +#[inline] +pub fn debug_enabled(level: i32) -> bool { + DEBUGLEVEL >= 2 && level <= unsafe { g_debuglevel } +} + +/// No-op when DEBUGLEVEL < 2; otherwise would log to stderr. +#[macro_export] +macro_rules! RAWLOG { + ($level:expr, $($arg:tt)*) => {{ + if $crate::debug::debug_enabled($level) { + eprint!($($arg)*); + } + }}; +} + +/// No-op when DEBUGLEVEL < 2; otherwise would log file:line prefix + message. +#[macro_export] +macro_rules! DEBUGLOG { + ($level:expr, $($arg:tt)*) => {{ + if $crate::debug::debug_enabled($level) { + eprint!("{}:{}: ", file!(), line!()); + eprintln!($($arg)*); + } + }}; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_build_has_logging_disabled() { + assert_eq!(DEBUGLEVEL, 0); + assert!(!debug_enabled(i32::MIN)); + } + + #[test] + fn logging_macros_accept_format_arguments() { + RAWLOG!(2, "{}", 1); + DEBUGLOG!(2, "{}", 1); + } +} diff --git a/rust/src/errors.rs b/rust/src/errors.rs new file mode 100644 index 000000000..44a3083cb --- /dev/null +++ b/rust/src/errors.rs @@ -0,0 +1,149 @@ +#![allow(non_snake_case)] +use std::os::raw::c_char; + +/// Matches `ZSTD_ErrorCode` / `ERR_enum` from the C API. +#[repr(i32)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum ZstdErrorCode { + NoError = 0, + Generic = 1, + PrefixUnknown = 10, + VersionUnsupported = 12, + FrameParameterUnsupported = 14, + FrameParameterWindowTooLarge = 16, + CorruptionDetected = 20, + ChecksumWrong = 22, + LiteralsHeaderWrong = 24, + DictionaryCorrupted = 30, + DictionaryWrong = 32, + DictionaryCreationFailed = 34, + ParameterUnsupported = 40, + ParameterCombinationUnsupported = 41, + ParameterOutOfBound = 42, + TableLogTooLarge = 44, + MaxSymbolValueTooLarge = 46, + MaxSymbolValueTooSmall = 48, + CannotProduceUncompressedBlock = 49, + StabilityConditionNotRespected = 50, + StageWrong = 60, + InitMissing = 62, + MemoryAllocation = 64, + WorkSpaceTooSmall = 66, + DstSizeTooSmall = 70, + SrcSizeWrong = 72, + DstBufferNull = 74, + NoForwardProgressDestFull = 80, + NoForwardProgressInputEmpty = 82, + FrameIndexTooLarge = 100, + SeekableIO = 102, + DstBufferWrong = 104, + SrcBufferWrong = 105, + SequenceProducerFailed = 106, + ExternalSequencesInvalid = 107, + MaxCode = 120, +} + +/// Encode an error as a size_t-style error code: `(size_t)-(code)`. +#[inline] +pub fn ERROR(code: ZstdErrorCode) -> usize { + (-(code as i32)) as usize +} + +/// Returns true if `code` is an error (same as C `ERR_isError`). +#[inline] +pub fn ERR_isError(code: usize) -> bool { + code > ERROR(ZstdErrorCode::MaxCode) +} + +/// Extract the numeric `ZSTD_ErrorCode` from a size_t error value. +/// +/// C permits error-shaped values which aren't named enum variants, and +/// `ERR_getErrorCode()` preserves those numeric values. Returning the ABI's +/// integer representation avoids constructing an invalid Rust enum. +#[inline] +pub fn ERR_getErrorCode(code: usize) -> i32 { + if !ERR_isError(code) { + return ZstdErrorCode::NoError as i32; + } + 0usize.wrapping_sub(code) as i32 +} + +#[inline] +pub fn ERR_getErrorName(code: usize) -> *const c_char { + ERR_getErrorString(ERR_getErrorCode(code)) +} + +/// C ABI: `ERR_getErrorString` (error_private.c). +#[no_mangle] +pub extern "C" fn ERR_getErrorString(code: i32) -> *const c_char { + match code { + 0 => c"No error detected".as_ptr(), + 1 => c"Error (generic)".as_ptr(), + 10 => c"Unknown frame descriptor".as_ptr(), + 12 => c"Version not supported".as_ptr(), + 14 => c"Unsupported frame parameter".as_ptr(), + 16 => c"Frame requires too much memory for decoding".as_ptr(), + 20 => c"Data corruption detected".as_ptr(), + 22 => c"Restored data doesn't match checksum".as_ptr(), + 24 => c"Header of Literals' block doesn't respect format specification".as_ptr(), + 30 => c"Dictionary is corrupted".as_ptr(), + 32 => c"Dictionary mismatch".as_ptr(), + 34 => c"Cannot create Dictionary from provided samples".as_ptr(), + 40 => c"Unsupported parameter".as_ptr(), + 41 => c"Unsupported combination of parameters".as_ptr(), + 42 => c"Parameter is out of bound".as_ptr(), + 44 => c"tableLog requires too much memory : unsupported".as_ptr(), + 46 => c"Unsupported max Symbol Value : too large".as_ptr(), + 48 => c"Specified maxSymbolValue is too small".as_ptr(), + 49 => c"This mode cannot generate an uncompressed block".as_ptr(), + 50 => c"pledged buffer stability condition is not respected".as_ptr(), + 60 => c"Operation not authorized at current processing stage".as_ptr(), + 62 => c"Context should be init first".as_ptr(), + 64 => c"Allocation error : not enough memory".as_ptr(), + 66 => c"workSpace buffer is not large enough".as_ptr(), + 70 => c"Destination buffer is too small".as_ptr(), + 72 => c"Src size is incorrect".as_ptr(), + 74 => c"Operation on NULL destination buffer".as_ptr(), + 80 => c"Operation made no progress over multiple calls, due to output buffer being full" + .as_ptr(), + 82 => c"Operation made no progress over multiple calls, due to input being empty".as_ptr(), + 100 => c"Frame index is too large".as_ptr(), + 102 => c"An I/O error occurred when reading/seeking".as_ptr(), + 104 => c"Destination buffer is wrong".as_ptr(), + 105 => c"Source buffer is wrong".as_ptr(), + 106 => c"Block-level external sequence producer returned an error code".as_ptr(), + 107 => c"External sequences are not valid".as_ptr(), + _ => c"Unspecified error code".as_ptr(), + } +} + +/// C ABI: public `ZSTD_getErrorString` from zstd_errors.h. +#[no_mangle] +pub extern "C" fn ZSTD_getErrorString(code: i32) -> *const c_char { + ERR_getErrorString(code) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CStr; + + #[test] + fn size_t_error_encoding_matches_c() { + let error = ERROR(ZstdErrorCode::DstSizeTooSmall); + assert!(ERR_isError(error)); + assert_eq!(ERR_getErrorCode(error), 70); + assert!(!ERR_isError(0)); + assert_eq!(ERR_getErrorCode(usize::MAX - 1), 2); + assert_eq!(ERR_getErrorCode(ERROR(ZstdErrorCode::MaxCode)), 0); + } + + #[test] + fn error_names_match_the_public_api() { + let name = unsafe { CStr::from_ptr(ERR_getErrorName(ERROR(ZstdErrorCode::ChecksumWrong))) }; + assert_eq!(name.to_bytes(), b"Restored data doesn't match checksum"); + + let unknown = unsafe { CStr::from_ptr(ERR_getErrorString(2)) }; + assert_eq!(unknown.to_bytes(), b"Unspecified error code"); + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs new file mode 100644 index 000000000..a282f3c27 --- /dev/null +++ b/rust/src/lib.rs @@ -0,0 +1,11 @@ +#![allow(clippy::missing_safety_doc)] + +pub mod bits; +pub mod bitstream; +pub mod common; +pub mod cpu; +pub mod debug; +pub mod errors; +pub mod mem; +pub mod xxhash; +pub mod zstd_common; diff --git a/rust/src/mem.rs b/rust/src/mem.rs new file mode 100644 index 000000000..8ff14630b --- /dev/null +++ b/rust/src/mem.rs @@ -0,0 +1,325 @@ +#![allow(non_snake_case)] +use std::mem; + +pub type U8 = u8; +pub type S8 = i8; +pub type U16 = u16; +pub type S16 = i16; +pub type U32 = u32; +pub type S32 = i32; +pub type U64 = u64; +pub type S64 = i64; +pub type BYTE = u8; + +#[inline] +pub fn MEM_32bits() -> bool { + mem::size_of::() == 4 +} + +#[inline] +pub fn MEM_64bits() -> bool { + mem::size_of::() == 8 +} + +#[inline] +pub fn MEM_isLittleEndian() -> bool { + cfg!(target_endian = "little") +} + +#[inline] +pub unsafe fn MEM_read16(mem_ptr: *const std::ffi::c_void) -> U16 { + (mem_ptr as *const U16).read_unaligned() +} + +#[inline] +pub unsafe fn MEM_read32(mem_ptr: *const std::ffi::c_void) -> U32 { + (mem_ptr as *const U32).read_unaligned() +} + +#[inline] +pub unsafe fn MEM_read64(mem_ptr: *const std::ffi::c_void) -> U64 { + (mem_ptr as *const U64).read_unaligned() +} + +#[inline] +pub unsafe fn MEM_readST(mem_ptr: *const std::ffi::c_void) -> usize { + (mem_ptr as *const usize).read_unaligned() +} + +#[inline] +pub unsafe fn MEM_write16(mem_ptr: *mut std::ffi::c_void, value: U16) { + (mem_ptr as *mut U16).write_unaligned(value); +} + +#[inline] +pub unsafe fn MEM_write32(mem_ptr: *mut std::ffi::c_void, value: U32) { + (mem_ptr as *mut U32).write_unaligned(value); +} + +#[inline] +pub unsafe fn MEM_write64(mem_ptr: *mut std::ffi::c_void, value: U64) { + (mem_ptr as *mut U64).write_unaligned(value); +} + +#[inline] +pub unsafe fn MEM_readLE16(mem_ptr: *const std::ffi::c_void) -> U16 { + U16::from_le(MEM_read16(mem_ptr)) +} + +#[inline] +pub unsafe fn MEM_readLE24(mem_ptr: *const std::ffi::c_void) -> U32 { + let low = MEM_readLE16(mem_ptr); + let high = *mem_ptr.cast::().add(2) as U32; + (low as U32) | (high << 16) +} + +#[inline] +pub unsafe fn MEM_readLE32(mem_ptr: *const std::ffi::c_void) -> U32 { + U32::from_le(MEM_read32(mem_ptr)) +} + +#[inline] +pub unsafe fn MEM_readLE64(mem_ptr: *const std::ffi::c_void) -> U64 { + U64::from_le(MEM_read64(mem_ptr)) +} + +#[inline] +pub unsafe fn MEM_readLEST(mem_ptr: *const std::ffi::c_void) -> usize { + if MEM_32bits() { + MEM_readLE32(mem_ptr) as usize + } else { + MEM_readLE64(mem_ptr) as usize + } +} + +#[inline] +pub unsafe fn MEM_writeLE16(mem_ptr: *mut std::ffi::c_void, val: U16) { + MEM_write16(mem_ptr, val.to_le()); +} + +#[inline] +pub unsafe fn MEM_writeLE24(mem_ptr: *mut std::ffi::c_void, val: U32) { + MEM_writeLE16(mem_ptr, val as U16); + *mem_ptr.cast::().add(2) = (val >> 16) as U8; +} + +#[inline] +pub unsafe fn MEM_writeLE32(mem_ptr: *mut std::ffi::c_void, val: U32) { + MEM_write32(mem_ptr, val.to_le()); +} + +#[inline] +pub unsafe fn MEM_writeLE64(mem_ptr: *mut std::ffi::c_void, val: U64) { + MEM_write64(mem_ptr, val.to_le()); +} + +#[inline] +pub unsafe fn MEM_writeLEST(mem_ptr: *mut std::ffi::c_void, val: usize) { + if MEM_32bits() { + MEM_writeLE32(mem_ptr, val as U32); + } else { + MEM_writeLE64(mem_ptr, val as U64); + } +} + +#[inline] +pub unsafe fn MEM_readBE32(mem_ptr: *const std::ffi::c_void) -> U32 { + U32::from_be(MEM_read32(mem_ptr)) +} + +#[inline] +pub unsafe fn MEM_writeBE32(mem_ptr: *mut std::ffi::c_void, val: U32) { + MEM_write32(mem_ptr, val.to_be()); +} + +#[inline] +pub unsafe fn MEM_readBE64(mem_ptr: *const std::ffi::c_void) -> U64 { + U64::from_be(MEM_read64(mem_ptr)) +} + +#[inline] +pub unsafe fn MEM_writeBE64(mem_ptr: *mut std::ffi::c_void, val: U64) { + MEM_write64(mem_ptr, val.to_be()); +} + +#[inline] +pub unsafe fn MEM_readBEST(mem_ptr: *const std::ffi::c_void) -> usize { + if MEM_32bits() { + MEM_readBE32(mem_ptr) as usize + } else { + MEM_readBE64(mem_ptr) as usize + } +} + +#[inline] +pub unsafe fn MEM_writeBEST(mem_ptr: *mut std::ffi::c_void, val: usize) { + if MEM_32bits() { + MEM_writeBE32(mem_ptr, val as U32); + } else { + MEM_writeBE64(mem_ptr, val as U64); + } +} + +#[inline] +pub fn MEM_swap32(in_val: U32) -> U32 { + in_val.swap_bytes() +} + +#[inline] +pub fn MEM_swap64(in_val: U64) -> U64 { + in_val.swap_bytes() +} + +#[inline] +pub fn MEM_swapST(in_val: usize) -> usize { + if MEM_32bits() { + MEM_swap32(in_val as U32) as usize + } else { + MEM_swap64(in_val as U64) as usize + } +} + +#[repr(C)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum ZstdOverlap { + NoOverlap, + OverlapSrcBeforeDst, +} + +/// # Safety +/// `dst`/`src` must be valid for the wildcopy contract (may over-read/write up to 32 bytes). +#[inline] +pub unsafe fn ZSTD_wildcopy( + dst: *mut std::ffi::c_void, + src: *const std::ffi::c_void, + length: isize, + ovtype: ZstdOverlap, +) { + const WILDCOPY_VECLEN: isize = 16; + let mut op = dst as *mut U8; + let mut ip = src as *const U8; + let oend = op.wrapping_offset(length); + let diff = (op as isize).wrapping_sub(ip as isize); + + if ovtype == ZstdOverlap::OverlapSrcBeforeDst && diff < WILDCOPY_VECLEN { + loop { + std::ptr::copy_nonoverlapping(ip, op, 8); + op = op.add(8); + ip = ip.add(8); + if op >= oend { + break; + } + } + } else { + std::ptr::copy_nonoverlapping(ip, op, 16); + if length <= 16 { + return; + } + op = op.add(16); + ip = ip.add(16); + while op < oend { + std::ptr::copy_nonoverlapping(ip, op, 16); + op = op.add(16); + ip = ip.add(16); + std::ptr::copy_nonoverlapping(ip, op, 16); + op = op.add(16); + ip = ip.add(16); + } + } +} + +/// # Safety +/// `dst` must be valid for `dst_capacity` bytes; `src` for `src_size` bytes. +#[inline] +pub unsafe fn ZSTD_limitCopy( + dst: *mut std::ffi::c_void, + dst_capacity: usize, + src: *const std::ffi::c_void, + src_size: usize, +) -> usize { + let length = std::cmp::min(dst_capacity, src_size); + if length > 0 { + std::ptr::copy_nonoverlapping(src as *const U8, dst as *mut U8, length); + } + length +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::c_void; + + #[test] + fn unaligned_endian_io_round_trips() { + let mut bytes = [0u8; 24]; + unsafe { + let p = bytes.as_mut_ptr().add(1).cast::(); + MEM_writeLE16(p, 0x1234); + assert_eq!(&bytes[1..3], &[0x34, 0x12]); + assert_eq!(MEM_readLE16(p), 0x1234); + + MEM_writeLE24(p, 0x00ab_cdef); + assert_eq!(&bytes[1..4], &[0xef, 0xcd, 0xab]); + assert_eq!(MEM_readLE24(p), 0x00ab_cdef); + + MEM_writeLE64(p, 0x0123_4567_89ab_cdef); + assert_eq!(MEM_readLE64(p), 0x0123_4567_89ab_cdef); + + MEM_writeBE64(p, 0x0123_4567_89ab_cdef); + assert_eq!(&bytes[1..9], &[1, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]); + assert_eq!(MEM_readBE64(p), 0x0123_4567_89ab_cdef); + } + } + + #[test] + fn wildcopy_always_performs_its_first_copy() { + let mut bytes = [0u8; 96]; + for (index, byte) in bytes[..32].iter_mut().enumerate() { + *byte = index as u8; + } + + unsafe { + ZSTD_wildcopy( + bytes.as_mut_ptr().add(64).cast(), + bytes.as_ptr().cast(), + 0, + ZstdOverlap::NoOverlap, + ); + } + assert_eq!(&bytes[64..80], &bytes[..16]); + } + + #[test] + fn short_offset_wildcopy_is_do_while_like() { + let mut bytes = [0u8; 32]; + for (index, byte) in bytes[..8].iter_mut().enumerate() { + *byte = (index + 1) as u8; + } + + unsafe { + ZSTD_wildcopy( + bytes.as_mut_ptr().add(8).cast(), + bytes.as_ptr().cast(), + 0, + ZstdOverlap::OverlapSrcBeforeDst, + ); + } + assert_eq!(&bytes[8..16], &[1, 2, 3, 4, 5, 6, 7, 8]); + } + + #[test] + fn limit_copy_reports_the_bytes_written() { + let src = *b"abcdef"; + let mut dst = [0u8; 4]; + let copied = unsafe { + ZSTD_limitCopy( + dst.as_mut_ptr().cast(), + dst.len(), + src.as_ptr().cast(), + src.len(), + ) + }; + assert_eq!(copied, 4); + assert_eq!(&dst, b"abcd"); + } +} diff --git a/rust/src/xxhash.rs b/rust/src/xxhash.rs new file mode 100644 index 000000000..3de022516 --- /dev/null +++ b/rust/src/xxhash.rs @@ -0,0 +1,688 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] + +use std::os::raw::c_void; + +pub type XXH32_hash_t = u32; +pub type XXH64_hash_t = u64; +pub type XXH_errorcode = i32; + +pub const XXH_OK: XXH_errorcode = 0; +pub const XXH_ERROR: XXH_errorcode = 1; + +const PRIME32_1: u32 = 0x9e37_79b1; +const PRIME32_2: u32 = 0x85eb_ca77; +const PRIME32_3: u32 = 0xc2b2_ae3d; +const PRIME32_4: u32 = 0x27d4_eb2f; +const PRIME32_5: u32 = 0x1656_67b1; + +const PRIME64_1: u64 = 0x9e37_79b1_85eb_ca87; +const PRIME64_2: u64 = 0xc2b2_ae3d_27d4_eb4f; +const PRIME64_3: u64 = 0x1656_67b1_9e37_79f9; +const PRIME64_4: u64 = 0x85eb_ca77_c2b2_ae63; +const PRIME64_5: u64 = 0x27d4_eb2f_1656_67c5; + +#[inline] +fn XXH64_round(mut acc: u64, input: u64) -> u64 { + acc = acc.wrapping_add(input.wrapping_mul(PRIME64_2)); + acc = acc.rotate_left(31); + acc.wrapping_mul(PRIME64_1) +} + +#[inline] +fn XXH64_mergeRound(mut acc: u64, val: u64) -> u64 { + acc ^= XXH64_round(0, val); + acc.wrapping_mul(PRIME64_1).wrapping_add(PRIME64_4) +} + +#[inline] +fn XXH64_avalanche(mut hash: u64) -> u64 { + hash ^= hash >> 33; + hash = hash.wrapping_mul(PRIME64_2); + hash ^= hash >> 29; + hash = hash.wrapping_mul(PRIME64_3); + hash ^= hash >> 32; + hash +} + +#[inline] +fn read_le64(input: &[u8]) -> u64 { + u64::from_le_bytes(input[..8].try_into().unwrap()) +} + +#[inline] +fn read_le32(input: &[u8]) -> u32 { + u32::from_le_bytes(input[..4].try_into().unwrap()) +} + +#[inline] +fn XXH32_round(mut acc: u32, input: u32) -> u32 { + acc = acc.wrapping_add(input.wrapping_mul(PRIME32_2)); + acc = acc.rotate_left(13); + acc.wrapping_mul(PRIME32_1) +} + +#[inline] +fn XXH32_avalanche(mut hash: u32) -> u32 { + hash ^= hash >> 15; + hash = hash.wrapping_mul(PRIME32_2); + hash ^= hash >> 13; + hash = hash.wrapping_mul(PRIME32_3); + hash ^= hash >> 16; + hash +} + +#[inline] +fn process_stripe32(accumulators: &mut [u32; 4], stripe: &[u8]) { + accumulators[0] = XXH32_round(accumulators[0], read_le32(&stripe[0..])); + accumulators[1] = XXH32_round(accumulators[1], read_le32(&stripe[4..])); + accumulators[2] = XXH32_round(accumulators[2], read_le32(&stripe[8..])); + accumulators[3] = XXH32_round(accumulators[3], read_le32(&stripe[12..])); +} + +#[inline] +fn finalize32(mut hash: u32, mut input: &[u8]) -> u32 { + while input.len() >= 4 { + hash = hash.wrapping_add(read_le32(input).wrapping_mul(PRIME32_3)); + hash = hash.rotate_left(17).wrapping_mul(PRIME32_4); + input = &input[4..]; + } + for &byte in input { + hash = hash.wrapping_add(u32::from(byte).wrapping_mul(PRIME32_5)); + hash = hash.rotate_left(11).wrapping_mul(PRIME32_1); + } + XXH32_avalanche(hash) +} + +fn hash32_bytes(input: &[u8], seed: u32) -> u32 { + let mut offset = 0; + let mut hash; + if input.len() >= 16 { + let mut accumulators = [ + seed.wrapping_add(PRIME32_1).wrapping_add(PRIME32_2), + seed.wrapping_add(PRIME32_2), + seed, + seed.wrapping_sub(PRIME32_1), + ]; + while offset + 16 <= input.len() { + process_stripe32(&mut accumulators, &input[offset..offset + 16]); + offset += 16; + } + hash = accumulators[0] + .rotate_left(1) + .wrapping_add(accumulators[1].rotate_left(7)) + .wrapping_add(accumulators[2].rotate_left(12)) + .wrapping_add(accumulators[3].rotate_left(18)); + } else { + hash = seed.wrapping_add(PRIME32_5); + } + hash = hash.wrapping_add(input.len() as u32); + finalize32(hash, &input[offset..]) +} + +#[inline] +fn process_stripe(accumulators: &mut [u64; 4], stripe: &[u8]) { + accumulators[0] = XXH64_round(accumulators[0], read_le64(&stripe[0..])); + accumulators[1] = XXH64_round(accumulators[1], read_le64(&stripe[8..])); + accumulators[2] = XXH64_round(accumulators[2], read_le64(&stripe[16..])); + accumulators[3] = XXH64_round(accumulators[3], read_le64(&stripe[24..])); +} + +#[inline] +fn finalize(mut hash: u64, mut input: &[u8]) -> u64 { + while input.len() >= 8 { + let lane = XXH64_round(0, read_le64(input)); + hash ^= lane; + hash = hash + .rotate_left(27) + .wrapping_mul(PRIME64_1) + .wrapping_add(PRIME64_4); + input = &input[8..]; + } + + if input.len() >= 4 { + hash ^= u64::from(read_le32(input)).wrapping_mul(PRIME64_1); + hash = hash + .rotate_left(23) + .wrapping_mul(PRIME64_2) + .wrapping_add(PRIME64_3); + input = &input[4..]; + } + + for &byte in input { + hash ^= u64::from(byte).wrapping_mul(PRIME64_5); + hash = hash.rotate_left(11).wrapping_mul(PRIME64_1); + } + + XXH64_avalanche(hash) +} + +fn hash_bytes(input: &[u8], seed: u64) -> u64 { + let mut offset = 0; + let mut hash; + + if input.len() >= 32 { + let mut accumulators = [ + seed.wrapping_add(PRIME64_1).wrapping_add(PRIME64_2), + seed.wrapping_add(PRIME64_2), + seed, + seed.wrapping_sub(PRIME64_1), + ]; + while offset + 32 <= input.len() { + process_stripe(&mut accumulators, &input[offset..offset + 32]); + offset += 32; + } + + hash = accumulators[0] + .rotate_left(1) + .wrapping_add(accumulators[1].rotate_left(7)) + .wrapping_add(accumulators[2].rotate_left(12)) + .wrapping_add(accumulators[3].rotate_left(18)); + for accumulator in accumulators { + hash = XXH64_mergeRound(hash, accumulator); + } + } else { + hash = seed.wrapping_add(PRIME64_5); + } + + hash = hash.wrapping_add(input.len() as u64); + finalize(hash, &input[offset..]) +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct XXH32_state_t { + pub total_len_32: u32, + pub large_len: u32, + pub v: [u32; 4], + pub mem32: [u32; 4], + pub memsize: u32, + pub reserved: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct XXH32_canonical_t { + pub digest: [u8; 4], +} + +#[export_name = "ZSTD_XXH32"] +pub unsafe extern "C" fn XXH32( + input: *const c_void, + length: usize, + seed: XXH32_hash_t, +) -> XXH32_hash_t { + if input.is_null() { + if length != 0 { + return 0; + } + return hash32_bytes(&[], seed); + } + hash32_bytes(std::slice::from_raw_parts(input.cast::(), length), seed) +} + +#[export_name = "ZSTD_XXH32_createState"] +pub unsafe extern "C" fn XXH32_createState() -> *mut XXH32_state_t { + libc::malloc(std::mem::size_of::()).cast::() +} + +#[export_name = "ZSTD_XXH32_freeState"] +pub unsafe extern "C" fn XXH32_freeState(state: *mut XXH32_state_t) -> XXH_errorcode { + libc::free(state.cast::()); + XXH_OK +} + +#[export_name = "ZSTD_XXH32_copyState"] +pub unsafe extern "C" fn XXH32_copyState( + destination: *mut XXH32_state_t, + source: *const XXH32_state_t, +) { + std::ptr::copy_nonoverlapping(source, destination, 1); +} + +#[export_name = "ZSTD_XXH32_reset"] +pub unsafe extern "C" fn XXH32_reset( + state: *mut XXH32_state_t, + seed: XXH32_hash_t, +) -> XXH_errorcode { + if state.is_null() { + return XXH_ERROR; + } + + std::ptr::write_bytes(state, 0, 1); + let state = &mut *state; + state.v[0] = seed.wrapping_add(PRIME32_1).wrapping_add(PRIME32_2); + state.v[1] = seed.wrapping_add(PRIME32_2); + state.v[2] = seed; + state.v[3] = seed.wrapping_sub(PRIME32_1); + XXH_OK +} + +#[export_name = "ZSTD_XXH32_update"] +pub unsafe extern "C" fn XXH32_update( + state: *mut XXH32_state_t, + input: *const c_void, + length: usize, +) -> XXH_errorcode { + if state.is_null() { + return XXH_ERROR; + } + if input.is_null() { + return XXH_OK; + } + + let state = &mut *state; + let input = std::slice::from_raw_parts(input.cast::(), length); + state.total_len_32 = state.total_len_32.wrapping_add(length as u32); + state.large_len |= u32::from(length >= 16 || state.total_len_32 >= 16); + + let buffered = state.memsize as usize; + if buffered + length < 16 { + std::ptr::copy_nonoverlapping( + input.as_ptr(), + state.mem32.as_mut_ptr().cast::().add(buffered), + length, + ); + state.memsize += length as u32; + return XXH_OK; + } + + let mut offset = 0; + if buffered != 0 { + let fill = 16 - buffered; + std::ptr::copy_nonoverlapping( + input.as_ptr(), + state.mem32.as_mut_ptr().cast::().add(buffered), + fill, + ); + let stripe = std::slice::from_raw_parts(state.mem32.as_ptr().cast::(), 16); + process_stripe32(&mut state.v, stripe); + offset = fill; + state.memsize = 0; + } + + while offset + 16 <= length { + process_stripe32(&mut state.v, &input[offset..offset + 16]); + offset += 16; + } + + let remaining = length - offset; + if remaining != 0 { + std::ptr::copy_nonoverlapping( + input.as_ptr().add(offset), + state.mem32.as_mut_ptr().cast::(), + remaining, + ); + state.memsize = remaining as u32; + } + XXH_OK +} + +#[export_name = "ZSTD_XXH32_digest"] +pub unsafe extern "C" fn XXH32_digest(state: *const XXH32_state_t) -> XXH32_hash_t { + if state.is_null() { + return 0; + } + let state = &*state; + let mut hash = if state.large_len != 0 { + state.v[0] + .rotate_left(1) + .wrapping_add(state.v[1].rotate_left(7)) + .wrapping_add(state.v[2].rotate_left(12)) + .wrapping_add(state.v[3].rotate_left(18)) + } else { + state.v[2].wrapping_add(PRIME32_5) + }; + hash = hash.wrapping_add(state.total_len_32); + let buffered = + std::slice::from_raw_parts(state.mem32.as_ptr().cast::(), state.memsize as usize); + finalize32(hash, buffered) +} + +#[export_name = "ZSTD_XXH32_canonicalFromHash"] +pub unsafe extern "C" fn XXH32_canonicalFromHash( + destination: *mut XXH32_canonical_t, + hash: XXH32_hash_t, +) { + (*destination).digest = hash.to_be_bytes(); +} + +#[export_name = "ZSTD_XXH32_hashFromCanonical"] +pub unsafe extern "C" fn XXH32_hashFromCanonical(source: *const XXH32_canonical_t) -> XXH32_hash_t { + u32::from_be_bytes((*source).digest) +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct XXH64_state_t { + pub total_len: u64, + pub v: [u64; 4], + pub mem64: [u64; 4], + pub memsize: u32, + pub reserved32: u32, + pub reserved64: u64, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct XXH64_canonical_t { + pub digest: [u8; 8], +} + +#[export_name = "ZSTD_XXH64"] +pub unsafe extern "C" fn XXH64( + input: *const c_void, + length: usize, + seed: XXH64_hash_t, +) -> XXH64_hash_t { + if input.is_null() { + if length != 0 { + return 0; + } + return hash_bytes(&[], seed); + } + hash_bytes(std::slice::from_raw_parts(input.cast::(), length), seed) +} + +#[export_name = "ZSTD_XXH64_createState"] +pub unsafe extern "C" fn XXH64_createState() -> *mut XXH64_state_t { + libc::malloc(std::mem::size_of::()).cast::() +} + +#[export_name = "ZSTD_XXH64_freeState"] +pub unsafe extern "C" fn XXH64_freeState(state: *mut XXH64_state_t) -> XXH_errorcode { + libc::free(state.cast::()); + XXH_OK +} + +#[export_name = "ZSTD_XXH64_copyState"] +pub unsafe extern "C" fn XXH64_copyState( + destination: *mut XXH64_state_t, + source: *const XXH64_state_t, +) { + std::ptr::copy_nonoverlapping(source, destination, 1); +} + +#[export_name = "ZSTD_XXH64_reset"] +pub unsafe extern "C" fn XXH64_reset( + state: *mut XXH64_state_t, + seed: XXH64_hash_t, +) -> XXH_errorcode { + if state.is_null() { + return XXH_ERROR; + } + + std::ptr::write_bytes(state, 0, 1); + let state = &mut *state; + state.v[0] = seed.wrapping_add(PRIME64_1).wrapping_add(PRIME64_2); + state.v[1] = seed.wrapping_add(PRIME64_2); + state.v[2] = seed; + state.v[3] = seed.wrapping_sub(PRIME64_1); + XXH_OK +} + +#[export_name = "ZSTD_XXH64_update"] +pub unsafe extern "C" fn XXH64_update( + state: *mut XXH64_state_t, + input: *const c_void, + length: usize, +) -> XXH_errorcode { + if state.is_null() { + return XXH_ERROR; + } + if input.is_null() { + return XXH_OK; + } + + let state = &mut *state; + let input = std::slice::from_raw_parts(input.cast::(), length); + state.total_len = state.total_len.wrapping_add(length as u64); + + let buffered = state.memsize as usize; + if buffered + length < 32 { + std::ptr::copy_nonoverlapping( + input.as_ptr(), + state.mem64.as_mut_ptr().cast::().add(buffered), + length, + ); + state.memsize += length as u32; + return XXH_OK; + } + + let mut offset = 0; + if buffered != 0 { + let fill = 32 - buffered; + std::ptr::copy_nonoverlapping( + input.as_ptr(), + state.mem64.as_mut_ptr().cast::().add(buffered), + fill, + ); + let stripe = std::slice::from_raw_parts(state.mem64.as_ptr().cast::(), 32); + process_stripe(&mut state.v, stripe); + offset = fill; + state.memsize = 0; + } + + while offset + 32 <= length { + process_stripe(&mut state.v, &input[offset..offset + 32]); + offset += 32; + } + + let remaining = length - offset; + if remaining != 0 { + std::ptr::copy_nonoverlapping( + input.as_ptr().add(offset), + state.mem64.as_mut_ptr().cast::(), + remaining, + ); + state.memsize = remaining as u32; + } + + XXH_OK +} + +#[export_name = "ZSTD_XXH64_digest"] +pub unsafe extern "C" fn XXH64_digest(state: *const XXH64_state_t) -> XXH64_hash_t { + if state.is_null() { + return 0; + } + let state = &*state; + + let mut hash = if state.total_len >= 32 { + let mut hash = state.v[0] + .rotate_left(1) + .wrapping_add(state.v[1].rotate_left(7)) + .wrapping_add(state.v[2].rotate_left(12)) + .wrapping_add(state.v[3].rotate_left(18)); + for accumulator in state.v { + hash = XXH64_mergeRound(hash, accumulator); + } + hash + } else { + state.v[2].wrapping_add(PRIME64_5) + }; + + hash = hash.wrapping_add(state.total_len); + let buffered = + std::slice::from_raw_parts(state.mem64.as_ptr().cast::(), state.memsize as usize); + finalize(hash, buffered) +} + +#[export_name = "ZSTD_XXH64_canonicalFromHash"] +pub unsafe extern "C" fn XXH64_canonicalFromHash( + destination: *mut XXH64_canonical_t, + hash: XXH64_hash_t, +) { + (*destination).digest = hash.to_be_bytes(); +} + +#[export_name = "ZSTD_XXH64_hashFromCanonical"] +pub unsafe extern "C" fn XXH64_hashFromCanonical(source: *const XXH64_canonical_t) -> XXH64_hash_t { + u64::from_be_bytes((*source).digest) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::mem::{offset_of, size_of, zeroed}; + + fn hash(input: &[u8], seed: u64) -> u64 { + unsafe { XXH64(input.as_ptr().cast(), input.len(), seed) } + } + + fn hash32(input: &[u8], seed: u32) -> u32 { + unsafe { XXH32(input.as_ptr().cast(), input.len(), seed) } + } + + #[test] + fn state_layout_matches_xxhash_h() { + assert_eq!(size_of::(), 48); + assert_eq!(offset_of!(XXH32_state_t, total_len_32), 0); + assert_eq!(offset_of!(XXH32_state_t, v), 8); + assert_eq!(offset_of!(XXH32_state_t, mem32), 24); + assert_eq!(offset_of!(XXH32_state_t, memsize), 40); + assert_eq!(size_of::(), 88); + assert_eq!(offset_of!(XXH64_state_t, total_len), 0); + assert_eq!(offset_of!(XXH64_state_t, v), 8); + assert_eq!(offset_of!(XXH64_state_t, mem64), 40); + assert_eq!(offset_of!(XXH64_state_t, memsize), 72); + assert_eq!(offset_of!(XXH64_state_t, reserved64), 80); + } + + #[test] + fn reference_vectors_match_xxhash_0_8() { + let vectors: &[(&[u8], u64)] = &[ + (b"", 0xef46_db37_51d8_e999), + (b"a", 0xd24e_c4f1_a98c_6e5b), + (b"abc", 0x44bc_2cf5_ad77_0999), + (b"message digest", 0x066e_d728_fcee_b3be), + (b"abcdefghijklmnopqrstuvwxyz", 0xcfe1_f278_fa89_835c), + ( + b"1234567890123456789012345678901234567890", + 0x5f3a_f5e2_3eeb_431d, + ), + ]; + for &(input, expected) in vectors { + assert_eq!(hash(input, 0), expected, "input={input:?}"); + } + + let vectors32: &[(&[u8], u32)] = &[ + (b"", 0x02cc_5d05), + (b"a", 0x550d_7456), + (b"abc", 0x32d1_53ff), + (b"message digest", 0x7c94_8494), + (b"abcdefghijklmnopqrstuvwxyz", 0x63a1_4d5f), + (b"1234567890123456789012345678901234567890", 0x765d_8c05), + ]; + for &(input, expected) in vectors32 { + assert_eq!(hash32(input, 0), expected, "input={input:?}"); + } + } + + #[test] + fn streaming_matches_one_shot_across_every_buffer_boundary() { + let input: Vec = (0..257) + .map(|index| (index as u8).wrapping_mul(101).wrapping_add(17)) + .collect(); + + for &seed in &[0, 1, u64::MAX, 0x0123_4567_89ab_cdef] { + let expected = hash(&input, seed); + for chunk_size in 1..=65 { + let mut state: XXH64_state_t = unsafe { zeroed() }; + assert_eq!(unsafe { XXH64_reset(&mut state, seed) }, XXH_OK); + for chunk in input.chunks(chunk_size) { + assert_eq!( + unsafe { XXH64_update(&mut state, chunk.as_ptr().cast(), chunk.len()) }, + XXH_OK + ); + } + assert_eq!( + unsafe { XXH64_digest(&state) }, + expected, + "chunk={chunk_size}" + ); + } + } + + for &seed in &[0, 1, u32::MAX, 0x89ab_cdef] { + let expected = hash32(&input, seed); + for chunk_size in 1..=33 { + let mut state: XXH32_state_t = unsafe { zeroed() }; + assert_eq!(unsafe { XXH32_reset(&mut state, seed) }, XXH_OK); + for chunk in input.chunks(chunk_size) { + assert_eq!( + unsafe { XXH32_update(&mut state, chunk.as_ptr().cast(), chunk.len()) }, + XXH_OK + ); + } + assert_eq!( + unsafe { XXH32_digest(&state) }, + expected, + "chunk={chunk_size}" + ); + } + } + } + + #[test] + fn intermediate_digests_do_not_modify_state() { + let input: Vec = (0..96).map(|value| value as u8).collect(); + let mut state: XXH64_state_t = unsafe { zeroed() }; + assert_eq!(unsafe { XXH64_reset(&mut state, 7) }, XXH_OK); + for end in 0..=input.len() { + if end != 0 { + assert_eq!( + unsafe { XXH64_update(&mut state, input[end - 1..].as_ptr().cast(), 1) }, + XXH_OK + ); + } + assert_eq!(unsafe { XXH64_digest(&state) }, hash(&input[..end], 7)); + } + } + + #[test] + fn canonical_form_is_big_endian() { + let mut canonical32 = XXH32_canonical_t { digest: [0; 4] }; + unsafe { XXH32_canonicalFromHash(&mut canonical32, 0x0123_4567) }; + assert_eq!(canonical32.digest, [1, 0x23, 0x45, 0x67]); + assert_eq!( + unsafe { XXH32_hashFromCanonical(&canonical32) }, + 0x0123_4567 + ); + + let mut canonical = XXH64_canonical_t { digest: [0; 8] }; + unsafe { XXH64_canonicalFromHash(&mut canonical, 0x0123_4567_89ab_cdef) }; + assert_eq!( + canonical.digest, + [1, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef] + ); + assert_eq!( + unsafe { XXH64_hashFromCanonical(&canonical) }, + 0x0123_4567_89ab_cdef + ); + } + + #[test] + fn null_empty_input_is_supported() { + assert_eq!(unsafe { XXH32(std::ptr::null(), 0, 0) }, 0x02cc_5d05); + let mut state32: XXH32_state_t = unsafe { zeroed() }; + assert_eq!(unsafe { XXH32_reset(&mut state32, 0) }, XXH_OK); + assert_eq!( + unsafe { XXH32_update(&mut state32, std::ptr::null(), 0) }, + XXH_OK + ); + assert_eq!(unsafe { XXH32_digest(&state32) }, 0x02cc_5d05); + + assert_eq!( + unsafe { XXH64(std::ptr::null(), 0, 0) }, + 0xef46_db37_51d8_e999 + ); + let mut state: XXH64_state_t = unsafe { zeroed() }; + assert_eq!(unsafe { XXH64_reset(&mut state, 0) }, XXH_OK); + assert_eq!( + unsafe { XXH64_update(&mut state, std::ptr::null(), 0) }, + XXH_OK + ); + assert_eq!(unsafe { XXH64_digest(&state) }, 0xef46_db37_51d8_e999); + } +} diff --git a/rust/src/zstd_common.rs b/rust/src/zstd_common.rs new file mode 100644 index 000000000..64543bc96 --- /dev/null +++ b/rust/src/zstd_common.rs @@ -0,0 +1,63 @@ +#![allow(non_snake_case)] +use crate::errors::{ERR_getErrorCode, ERR_getErrorName, ERR_isError, ZstdErrorCode}; +use std::os::raw::{c_char, c_uint}; + +pub const ZSTD_VERSION_MAJOR: u32 = 1; +pub const ZSTD_VERSION_MINOR: u32 = 5; +pub const ZSTD_VERSION_RELEASE: u32 = 7; +pub const ZSTD_VERSION_NUMBER: u32 = + ZSTD_VERSION_MAJOR * 100 * 100 + ZSTD_VERSION_MINOR * 100 + ZSTD_VERSION_RELEASE; + +#[no_mangle] +pub extern "C" fn ZSTD_versionNumber() -> c_uint { + ZSTD_VERSION_NUMBER +} + +#[no_mangle] +pub extern "C" fn ZSTD_versionString() -> *const c_char { + c"1.5.7".as_ptr() +} + +#[no_mangle] +pub extern "C" fn ZSTD_isError(code: usize) -> c_uint { + ERR_isError(code) as c_uint +} + +#[no_mangle] +pub extern "C" fn ZSTD_getErrorName(code: usize) -> *const c_char { + ERR_getErrorName(code) +} + +#[no_mangle] +pub extern "C" fn ZSTD_getErrorCode(code: usize) -> i32 { + ERR_getErrorCode(code) +} + +// ZSTD_getErrorString lives in errors.rs (also used as ERR_getErrorString). + +/// Convenience re-export of the error enum for callers of the common module. +pub type ErrorCode = ZstdErrorCode; + +#[cfg(test)] +mod tests { + use super::*; + use crate::errors::{ZstdErrorCode, ERROR}; + use std::ffi::CStr; + + #[test] + fn version_exports_match_zstd_h() { + assert_eq!(ZSTD_versionNumber(), 10_507); + let version = unsafe { CStr::from_ptr(ZSTD_versionString()) }; + assert_eq!(version.to_bytes(), b"1.5.7"); + } + + #[test] + fn public_error_exports_preserve_numeric_codes() { + let error = ERROR(ZstdErrorCode::CorruptionDetected); + assert_eq!(ZSTD_isError(error), 1); + assert_eq!(ZSTD_getErrorCode(error), 20); + assert_eq!(ZSTD_getErrorCode(usize::MAX - 1), 2); + let name = unsafe { CStr::from_ptr(ZSTD_getErrorName(error)) }; + assert_eq!(name.to_bytes(), b"Data corruption detected"); + } +} diff --git a/tests/Makefile b/tests/Makefile index 406c7f20b..b604e6eba 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -74,6 +74,7 @@ MULTITHREAD_CPP = -DZSTD_MULTITHREAD MULTITHREAD_LD = -pthread endif MULTITHREAD = $(MULTITHREAD_CPP) $(MULTITHREAD_LD) +LDFLAGS += -Wl,--whole-archive ../rust/target/release/libzstd_rs.a -Wl,--no-whole-archive VOID = /dev/null ZSTREAM_TESTTIME ?= -T90s