From 5f9d607ddd4e38db3c6e48ffae1da5d754fb6d76 Mon Sep 17 00:00:00 2001 From: ddidderr Date: Fri, 10 Jul 2026 20:32:18 +0200 Subject: [PATCH] feat(rust): port FSE compression primitives Move FSE normalization, count-header writing, compression-table construction, and stream encoding from C to Rust. The original C source remains as a header-only shim, preserving the public header configuration while native consumers receive the Rust archive exports. This completes both codec directions for FSE in the migration crate and lets the unchanged C compatibility tests exercise the Rust encoder implementation. Test Plan: - cargo clippy; cargo clippy --benches; cargo clippy --tests - cargo +nightly fmt, then repeat the Clippy checks - cargo test --all-targets - cargo build --release - Compile the C shim with strict warnings enabled - 1,000-case pristine-C differential for tables, headers, and payloads - ./tests/fuzzer -v -T10s - ./tests/decodecorpus -t -T5 Refs: rust/README.md --- lib/compress/fse_compress.c | 623 +------------------- rust/README.md | 2 + rust/src/fse_compress.rs | 1085 +++++++++++++++++++++++++++++++++++ rust/src/lib.rs | 1 + 4 files changed, 1089 insertions(+), 622 deletions(-) create mode 100644 rust/src/fse_compress.rs diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c index 1ce3cf16a..78123b01e 100644 --- a/lib/compress/fse_compress.c +++ b/lib/compress/fse_compress.c @@ -1,625 +1,4 @@ -/* ****************************************************************** - * FSE : Finite State Entropy encoder - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * You can contact the author at : - * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy - * - Public forum : https://groups.google.com/forum/#!forum/lz4c - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. -****************************************************************** */ - -/* ************************************************************** -* Includes -****************************************************************/ -#include "../common/compiler.h" -#include "../common/mem.h" /* U32, U16, etc. */ -#include "../common/debug.h" /* assert, DEBUGLOG */ -#include "hist.h" /* HIST_count_wksp */ -#include "../common/bitstream.h" #define FSE_STATIC_LINKING_ONLY #include "../common/fse.h" -#include "../common/error_private.h" -#define ZSTD_DEPS_NEED_MALLOC -#define ZSTD_DEPS_NEED_MATH64 -#include "../common/zstd_deps.h" /* ZSTD_memset */ -#include "../common/bits.h" /* ZSTD_highbit32 */ - -/* ************************************************************** -* Error Management -****************************************************************/ -#define FSE_isError ERR_isError - - -/* ************************************************************** -* Templates -****************************************************************/ -/* - designed to be included - for type-specific functions (template emulation in C) - Objective is to write these functions only once, for improved maintenance -*/ - -/* safety checks */ -#ifndef FSE_FUNCTION_EXTENSION -# error "FSE_FUNCTION_EXTENSION must be defined" -#endif -#ifndef FSE_FUNCTION_TYPE -# error "FSE_FUNCTION_TYPE must be defined" -#endif - -/* Function names */ -#define FSE_CAT(X,Y) X##Y -#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) -#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) - - -/* Function templates */ - -/* FSE_buildCTable_wksp() : - * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). - * wkspSize should be sized to handle worst case situation, which is `1<>1 : 1) ; - FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); - U32 const step = FSE_TABLESTEP(tableSize); - U32 const maxSV1 = maxSymbolValue+1; - - U16* cumul = (U16*)workSpace; /* size = maxSV1 */ - FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)(cumul + (maxSV1+1)); /* size = tableSize */ - - U32 highThreshold = tableSize-1; - - assert(((size_t)workSpace & 1) == 0); /* Must be 2 bytes-aligned */ - if (FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) > wkspSize) return ERROR(tableLog_tooLarge); - /* CTable header */ - tableU16[-2] = (U16) tableLog; - tableU16[-1] = (U16) maxSymbolValue; - assert(tableLog < 16); /* required for threshold strategy to work */ - - /* For explanations on how to distribute symbol values over the table : - * https://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ - - #ifdef __clang_analyzer__ - ZSTD_memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */ - #endif - - /* symbol start positions */ - { U32 u; - cumul[0] = 0; - for (u=1; u <= maxSV1; u++) { - if (normalizedCounter[u-1]==-1) { /* Low proba symbol */ - cumul[u] = cumul[u-1] + 1; - tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1); - } else { - assert(normalizedCounter[u-1] >= 0); - cumul[u] = cumul[u-1] + (U16)normalizedCounter[u-1]; - assert(cumul[u] >= cumul[u-1]); /* no overflow */ - } } - cumul[maxSV1] = (U16)(tableSize+1); - } - - /* Spread symbols */ - if (highThreshold == tableSize - 1) { - /* Case for no low prob count symbols. Lay down 8 bytes at a time - * to reduce branch misses since we are operating on a small block - */ - BYTE* const spread = tableSymbol + tableSize; /* size = tableSize + 8 (may write beyond tableSize) */ - { U64 const add = 0x0101010101010101ull; - size_t pos = 0; - U64 sv = 0; - U32 s; - for (s=0; s=0); - pos += (size_t)n; - } - } - /* Spread symbols across the table. Lack of lowprob symbols means that - * we don't need variable sized inner loop, so we can unroll the loop and - * reduce branch misses. - */ - { size_t position = 0; - size_t s; - size_t const unroll = 2; /* Experimentally determined optimal unroll */ - assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */ - for (s = 0; s < (size_t)tableSize; s += unroll) { - size_t u; - for (u = 0; u < unroll; ++u) { - size_t const uPosition = (position + (u * step)) & tableMask; - tableSymbol[uPosition] = spread[s + u]; - } - position = (position + (unroll * step)) & tableMask; - } - assert(position == 0); /* Must have initialized all positions */ - } - } else { - U32 position = 0; - U32 symbol; - for (symbol=0; symbol highThreshold) - position = (position + step) & tableMask; /* Low proba area */ - } } - assert(position==0); /* Must have initialized all positions */ - } - - /* Build table */ - { U32 u; for (u=0; u 1); - { U32 const maxBitsOut = tableLog - ZSTD_highbit32 ((U32)normalizedCounter[s]-1); - U32 const minStatePlus = (U32)normalizedCounter[s] << maxBitsOut; - symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus; - symbolTT[s].deltaFindState = (int)(total - (unsigned)normalizedCounter[s]); - total += (unsigned)normalizedCounter[s]; - } } } } - -#if 0 /* debug : symbol costs */ - DEBUGLOG(5, "\n --- table statistics : "); - { U32 symbol; - for (symbol=0; symbol<=maxSymbolValue; symbol++) { - DEBUGLOG(5, "%3u: w=%3i, maxBits=%u, fracBits=%.2f", - symbol, normalizedCounter[symbol], - FSE_getMaxNbBits(symbolTT, symbol), - (double)FSE_bitCost(symbolTT, tableLog, symbol, 8) / 256); - } } -#endif - - return 0; -} - - - -#ifndef FSE_COMMONDEFS_ONLY - -/*-************************************************************** -* FSE NCount encoding -****************************************************************/ -size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog) -{ - size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog - + 4 /* bitCount initialized at 4 */ - + 2 /* first two symbols may use one additional bit each */) / 8) - + 1 /* round up to whole nb bytes */ - + 2 /* additional two bytes for bitstream flush */; - return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ -} - -static size_t -FSE_writeNCount_generic (void* header, size_t headerBufferSize, - const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, - unsigned writeIsSafe) -{ - BYTE* const ostart = (BYTE*) header; - BYTE* out = ostart; - BYTE* const oend = ostart + headerBufferSize; - int nbBits; - const int tableSize = 1 << tableLog; - int remaining; - int threshold; - U32 bitStream = 0; - int bitCount = 0; - unsigned symbol = 0; - unsigned const alphabetSize = maxSymbolValue + 1; - int previousIs0 = 0; - - /* Table Size */ - bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount; - bitCount += 4; - - /* Init */ - remaining = tableSize+1; /* +1 for extra accuracy */ - threshold = tableSize; - nbBits = (int)tableLog+1; - - while ((symbol < alphabetSize) && (remaining>1)) { /* stops at 1 */ - if (previousIs0) { - unsigned start = symbol; - while ((symbol < alphabetSize) && !normalizedCounter[symbol]) symbol++; - if (symbol == alphabetSize) break; /* incorrect distribution */ - while (symbol >= start+24) { - start+=24; - bitStream += 0xFFFFU << bitCount; - if ((!writeIsSafe) && (out > oend-2)) - return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE) bitStream; - out[1] = (BYTE)(bitStream>>8); - out+=2; - bitStream>>=16; - } - while (symbol >= start+3) { - start+=3; - bitStream += 3U << bitCount; - bitCount += 2; - } - bitStream += (symbol-start) << bitCount; - bitCount += 2; - if (bitCount>16) { - if ((!writeIsSafe) && (out > oend - 2)) - return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream>>8); - out += 2; - bitStream >>= 16; - bitCount -= 16; - } } - { int count = normalizedCounter[symbol++]; - int const max = (2*threshold-1) - remaining; - remaining -= count < 0 ? -count : count; - count++; /* +1 for extra accuracy */ - if (count>=threshold) - count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ - bitStream += (U32)count << bitCount; - bitCount += nbBits; - bitCount -= (count>=1; } - } - if (bitCount>16) { - if ((!writeIsSafe) && (out > oend - 2)) - return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream>>8); - out += 2; - bitStream >>= 16; - bitCount -= 16; - } } - - if (remaining != 1) - return ERROR(GENERIC); /* incorrect normalized distribution */ - assert(symbol <= alphabetSize); - - /* flush remaining bitStream */ - if ((!writeIsSafe) && (out > oend - 2)) - return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream>>8); - out+= (bitCount+7) /8; - - assert(out >= ostart); - return (size_t)(out-ostart); -} - - -size_t FSE_writeNCount (void* buffer, size_t bufferSize, - const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) -{ - if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */ - if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */ - - if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) - return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); - - return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1 /* write in buffer is safe */); -} - - -/*-************************************************************** -* FSE Compression Code -****************************************************************/ - -/* provides the minimum logSize to safely represent a distribution */ -static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) -{ - U32 minBitsSrc = ZSTD_highbit32((U32)(srcSize)) + 1; - U32 minBitsSymbols = ZSTD_highbit32(maxSymbolValue) + 2; - U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; - assert(srcSize > 1); /* Not supported, RLE should be used instead */ - return minBits; -} - -unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) -{ - U32 maxBitsSrc = ZSTD_highbit32((U32)(srcSize - 1)) - minus; - U32 tableLog = maxTableLog; - U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); - assert(srcSize > 1); /* Not supported, RLE should be used instead */ - if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; - if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ - if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ - if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; - if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; - return tableLog; -} - -unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) -{ - return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); -} - -/* Secondary normalization method. - To be used when primary method fails. */ - -static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue, short lowProbCount) -{ - short const NOT_YET_ASSIGNED = -2; - U32 s; - U32 distributed = 0; - U32 ToDistribute; - - /* Init */ - U32 const lowThreshold = (U32)(total >> tableLog); - U32 lowOne = (U32)((total * 3) >> (tableLog + 1)); - - for (s=0; s<=maxSymbolValue; s++) { - if (count[s] == 0) { - norm[s]=0; - continue; - } - if (count[s] <= lowThreshold) { - norm[s] = lowProbCount; - distributed++; - total -= count[s]; - continue; - } - if (count[s] <= lowOne) { - norm[s] = 1; - distributed++; - total -= count[s]; - continue; - } - - norm[s]=NOT_YET_ASSIGNED; - } - ToDistribute = (1 << tableLog) - distributed; - - if (ToDistribute == 0) - return 0; - - if ((total / ToDistribute) > lowOne) { - /* risk of rounding to zero */ - lowOne = (U32)((total * 3) / (ToDistribute * 2)); - for (s=0; s<=maxSymbolValue; s++) { - if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) { - norm[s] = 1; - distributed++; - total -= count[s]; - continue; - } } - ToDistribute = (1 << tableLog) - distributed; - } - - if (distributed == maxSymbolValue+1) { - /* all values are pretty poor; - probably incompressible data (should have already been detected); - find max, then give all remaining points to max */ - U32 maxV = 0, maxC = 0; - for (s=0; s<=maxSymbolValue; s++) - if (count[s] > maxC) { maxV=s; maxC=count[s]; } - norm[maxV] += (short)ToDistribute; - return 0; - } - - if (total == 0) { - /* all of the symbols were low enough for the lowOne or lowThreshold */ - for (s=0; ToDistribute > 0; s = (s+1)%(maxSymbolValue+1)) - if (norm[s] > 0) { ToDistribute--; norm[s]++; } - return 0; - } - - { U64 const vStepLog = 62 - tableLog; - U64 const mid = (1ULL << (vStepLog-1)) - 1; - U64 const rStep = ZSTD_div64((((U64)1<> vStepLog); - U32 const sEnd = (U32)(end >> vStepLog); - U32 const weight = sEnd - sStart; - if (weight < 1) - return ERROR(GENERIC); - norm[s] = (short)weight; - tmpTotal = end; - } } } - - return 0; -} - -size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, - const unsigned* count, size_t total, - unsigned maxSymbolValue, unsigned useLowProbCount) -{ - /* Sanity checks */ - if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; - if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */ - if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */ - if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ - - { static U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 }; - short const lowProbCount = useLowProbCount ? -1 : 1; - U64 const scale = 62 - tableLog; - U64 const step = ZSTD_div64((U64)1<<62, (U32)total); /* <== here, one division ! */ - U64 const vStep = 1ULL<<(scale-20); - int stillToDistribute = 1<> tableLog); - - for (s=0; s<=maxSymbolValue; s++) { - if (count[s] == total) return 0; /* rle special case */ - if (count[s] == 0) { normalizedCounter[s]=0; continue; } - if (count[s] <= lowThreshold) { - normalizedCounter[s] = lowProbCount; - stillToDistribute--; - } else { - short proba = (short)((count[s]*step) >> scale); - if (proba<8) { - U64 restToBeat = vStep * rtbTable[proba]; - proba += (count[s]*step) - ((U64)proba< restToBeat; - } - if (proba > largestP) { largestP=proba; largest=s; } - normalizedCounter[s] = proba; - stillToDistribute -= proba; - } } - if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { - /* corner case, need another normalization method */ - size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue, lowProbCount); - if (FSE_isError(errorCode)) return errorCode; - } - else normalizedCounter[largest] += (short)stillToDistribute; - } - -#if 0 - { /* Print Table (debug) */ - U32 s; - U32 nTotal = 0; - for (s=0; s<=maxSymbolValue; s++) - RAWLOG(2, "%3i: %4i \n", s, normalizedCounter[s]); - for (s=0; s<=maxSymbolValue; s++) - nTotal += abs(normalizedCounter[s]); - if (nTotal != (1U< FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */ - FSE_encodeSymbol(&bitC, &CState2, *--ip); - FSE_encodeSymbol(&bitC, &CState1, *--ip); - FSE_FLUSHBITS(&bitC); - } - - /* 2 or 4 encoding per loop */ - while ( ip>istart ) { - - FSE_encodeSymbol(&bitC, &CState2, *--ip); - - if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */ - FSE_FLUSHBITS(&bitC); - - FSE_encodeSymbol(&bitC, &CState1, *--ip); - - if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */ - FSE_encodeSymbol(&bitC, &CState2, *--ip); - FSE_encodeSymbol(&bitC, &CState1, *--ip); - } - - FSE_FLUSHBITS(&bitC); - } - - FSE_flushCState(&bitC, &CState2); - FSE_flushCState(&bitC, &CState1); - return BIT_closeCStream(&bitC); -} - -size_t FSE_compress_usingCTable (void* dst, size_t dstSize, - const void* src, size_t srcSize, - const FSE_CTable* ct) -{ - unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize)); - - if (fast) - return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1); - else - return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); -} - - -size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } - -#endif /* FSE_COMMONDEFS_ONLY */ +/* Implementation moved to Rust (rust/src/fse_compress.rs). */ diff --git a/rust/README.md b/rust/README.md index c587662cb..1038e2ac7 100644 --- a/rust/README.md +++ b/rust/README.md @@ -20,6 +20,8 @@ zstd ABI: - Entropy coding - `entropy_common` reads FSE normalized counts and Huffman statistics. - `fse_decompress` builds FSE decoding tables and decodes FSE streams. + - `fse_compress` normalizes counts, writes FSE headers, builds compression + tables, and encodes FSE streams. - Compression primitives - `hist` counts byte frequencies for FSE and Huffman compression. - Runtime support diff --git a/rust/src/fse_compress.rs b/rust/src/fse_compress.rs new file mode 100644 index 000000000..feba9d7d6 --- /dev/null +++ b/rust/src/fse_compress.rs @@ -0,0 +1,1085 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] + +use crate::bitstream::{ + BIT_CStream_t, BIT_addBits, BIT_closeCStream, BIT_flushBits, BIT_flushBitsFast, BIT_initCStream, +}; +use crate::entropy_common::FSE_MIN_TABLELOG; +use crate::errors::{ERR_isError, ZstdErrorCode, ERROR}; +use std::mem::size_of; +use std::os::raw::{c_int, c_short, c_uint, c_void}; + +pub const FSE_DEFAULT_TABLELOG: u32 = 11; +pub const FSE_NCOUNTBOUND: usize = 512; +pub const FSE_MAX_TABLELOG: u32 = 12; +const FSE_TABLELOG_ABSOLUTE_MAX: u32 = 15; + +#[repr(C)] +#[derive(Clone, Copy)] +struct FSE_symbolCompressionTransform { + deltaFindState: c_int, + deltaNbBits: u32, +} + +#[repr(C)] +struct FSE_CState_t { + value: isize, + stateTable: *const c_void, + symbolTT: *const c_void, + stateLog: u32, +} + +#[inline] +fn FSE_TABLESTEP(table_size: usize) -> usize { + (table_size >> 1) + (table_size >> 3) + 3 +} + +#[inline] +fn fse_table_size(table_log: u32) -> Option { + if table_log > FSE_TABLELOG_ABSOLUTE_MAX || table_log >= usize::BITS { + return None; + } + Some(1usize << table_log) +} + +#[inline] +fn fse_ctable_transform_offset(table_log: u32) -> usize { + // `((U32*)ct) + 1 + (tableLog ? tableSize >> 1 : 1)` in fse.h. + 4 + if table_log == 0 { + 4 + } else { + (1usize << table_log) * 2 + } +} + +#[inline] +fn fse_build_ctable_workspace_size(max_symbol_value: u32, table_log: u32) -> Option { + let table_size = fse_table_size(table_log)?; + let words = (max_symbol_value as usize) + .checked_add(2)? + .checked_add(table_size)? + / 2 + + size_of::() / size_of::(); + words.checked_mul(size_of::()) +} + +#[inline] +unsafe fn fse_read_u16(base: *const u8, index: usize) -> u16 { + (base.add(index * size_of::()) as *const u16).read_unaligned() +} + +#[inline] +unsafe fn fse_write_u16(base: *mut u8, index: usize, value: u16) { + (base.add(index * size_of::()) as *mut u16).write_unaligned(value); +} + +#[inline] +unsafe fn fse_read_transform( + ctable: *const c_uint, + table_log: u32, + symbol: usize, +) -> FSE_symbolCompressionTransform { + let base = ctable as *const u8; + (base.add( + fse_ctable_transform_offset(table_log) + + symbol * size_of::(), + ) as *const FSE_symbolCompressionTransform) + .read_unaligned() +} + +#[inline] +unsafe fn fse_write_transform( + ctable: *mut c_uint, + table_log: u32, + symbol: usize, + transform: FSE_symbolCompressionTransform, +) { + let base = ctable as *mut u8; + (base.add( + fse_ctable_transform_offset(table_log) + + symbol * size_of::(), + ) as *mut FSE_symbolCompressionTransform) + .write_unaligned(transform); +} + +#[inline] +unsafe fn fse_read_normalized(normalized_counter: *const c_short, symbol: u32) -> i16 { + *normalized_counter.add(symbol as usize) +} + +#[inline] +unsafe fn fse_read_count(count: *const c_uint, symbol: u32) -> u32 { + *count.add(symbol as usize) +} + +#[inline] +unsafe fn fse_write_normalized(normalized_counter: *mut c_short, symbol: u32, value: i16) { + *normalized_counter.add(symbol as usize) = value; +} + +#[no_mangle] +pub unsafe extern "C" fn FSE_buildCTable_wksp( + ct: *mut c_uint, + normalized_counter: *const c_short, + max_symbol_value: c_uint, + table_log: c_uint, + work_space: *mut c_void, + wksp_size: usize, +) -> usize { + let table_size = match fse_table_size(table_log) { + Some(size) => size, + None => return ERROR(ZstdErrorCode::TableLogTooLarge), + }; + let required_wksp = match fse_build_ctable_workspace_size(max_symbol_value, table_log) { + Some(size) => size, + None => return ERROR(ZstdErrorCode::TableLogTooLarge), + }; + if required_wksp > wksp_size { + return ERROR(ZstdErrorCode::TableLogTooLarge); + } + + let table_mask = table_size - 1; + let max_sv1 = max_symbol_value as usize + 1; + let ctable = ct as *mut u8; + let workspace = work_space as *mut u8; + let cumul = workspace; + let table_symbol = cumul.add((max_sv1 + 1) * size_of::()); + let mut high_threshold = table_size - 1; + + // CTable header: tableLog and maxSymbolValue are two native-endian U16s. + fse_write_u16(ctable, 0, table_log as u16); + fse_write_u16(ctable, 1, max_symbol_value as u16); + + fse_write_u16(cumul, 0, 0); + for symbol_plus_one in 1..=max_sv1 { + let symbol = (symbol_plus_one - 1) as u32; + let previous = fse_read_u16(cumul, symbol_plus_one - 1); + let normalized = fse_read_normalized(normalized_counter, symbol); + if normalized == -1 { + fse_write_u16(cumul, symbol_plus_one, previous.wrapping_add(1)); + *table_symbol.add(high_threshold) = symbol as u8; + high_threshold = high_threshold.wrapping_sub(1); + } else { + debug_assert!(normalized >= 0); + fse_write_u16( + cumul, + symbol_plus_one, + previous.wrapping_add(normalized as u16), + ); + } + } + fse_write_u16(cumul, max_sv1, (table_size + 1) as u16); + + if high_threshold == table_size - 1 { + // The C implementation stores the sequential symbol list immediately + // past tableSymbol, then permutes it into tableSymbol. Keep that + // workspace layout so callers can provide exactly the C macro size. + let spread = table_symbol.add(table_size); + let mut position = 0usize; + for symbol in 0..max_sv1 { + let frequency = fse_read_normalized(normalized_counter, symbol as u32); + debug_assert!(frequency >= 0); + for offset in 0..frequency as usize { + *spread.add(position + offset) = symbol as u8; + } + position += frequency as usize; + } + + let step = FSE_TABLESTEP(table_size); + let mut position = 0usize; + for source_index in (0..table_size).step_by(2) { + for unroll in 0..2 { + let target = (position + unroll * step) & table_mask; + *table_symbol.add(target) = *spread.add(source_index + unroll); + } + position = (position + 2 * step) & table_mask; + } + debug_assert_eq!(position, 0); + } else { + let step = FSE_TABLESTEP(table_size); + let mut position = 0usize; + for symbol in 0..max_sv1 { + let frequency = fse_read_normalized(normalized_counter, symbol as u32); + for _ in 0..frequency.max(0) as usize { + *table_symbol.add(position) = symbol as u8; + position = (position + step) & table_mask; + while position > high_threshold { + position = (position + step) & table_mask; + } + } + } + debug_assert_eq!(position, 0); + } + + // tableU16 is `(U16*)ct + 2`, so its first entry is U16 index 2. + for table_index in 0..table_size { + let symbol = *table_symbol.add(table_index) as usize; + let current = fse_read_u16(cumul, symbol); + fse_write_u16( + ctable, + 2 + current as usize, + (table_size + table_index) as u16, + ); + fse_write_u16(cumul, symbol, current.wrapping_add(1)); + } + + let mut total = 0u32; + for symbol in 0..=max_symbol_value { + let normalized = fse_read_normalized(normalized_counter, symbol); + let transform = match normalized { + 0 => FSE_symbolCompressionTransform { + deltaFindState: 0, + deltaNbBits: ((table_log + 1) << 16).wrapping_sub(1u32 << table_log), + }, + -1 | 1 => { + let transform = FSE_symbolCompressionTransform { + deltaFindState: total.wrapping_sub(1) as i32, + deltaNbBits: (table_log << 16).wrapping_sub(1u32 << table_log), + }; + total = total.wrapping_add(1); + transform + } + _ => { + debug_assert!(normalized > 1); + let normalized = normalized as u32; + let max_bits_out = table_log - (31 - (normalized - 1).leading_zeros()); + let min_state_plus = normalized << max_bits_out; + let transform = FSE_symbolCompressionTransform { + deltaFindState: total.wrapping_sub(normalized) as i32, + deltaNbBits: (max_bits_out << 16).wrapping_sub(min_state_plus), + }; + total = total.wrapping_add(normalized); + transform + } + }; + fse_write_transform(ct, table_log, symbol as usize, transform); + } + + 0 +} + +#[no_mangle] +pub extern "C" fn FSE_NCountWriteBound(max_symbol_value: c_uint, table_log: c_uint) -> usize { + if max_symbol_value == 0 { + return FSE_NCOUNTBOUND; + } + (((max_symbol_value as usize + 1) * table_log as usize + 4 + 2) / 8) + 1 + 2 +} + +unsafe fn fse_write_two_bytes( + buffer: *mut u8, + buffer_size: usize, + out_pos: &mut usize, + bit_stream: u32, +) -> Result<(), usize> { + if buffer_size.saturating_sub(*out_pos) < 2 { + return Err(ERROR(ZstdErrorCode::DstSizeTooSmall)); + } + *buffer.add(*out_pos) = bit_stream as u8; + *buffer.add(*out_pos + 1) = (bit_stream >> 8) as u8; + *out_pos += 2; + Ok(()) +} + +unsafe fn FSE_writeNCount_generic( + header: *mut c_void, + header_buffer_size: usize, + normalized_counter: *const c_short, + max_symbol_value: c_uint, + table_log: c_uint, +) -> usize { + let buffer = header as *mut u8; + let table_size = 1i32 << table_log; + let mut bit_stream = table_log - FSE_MIN_TABLELOG; + let mut bit_count = 4i32; + let mut remaining = table_size + 1; + let mut threshold = table_size; + let mut nb_bits = table_log as i32 + 1; + let mut symbol = 0u32; + let alphabet_size = max_symbol_value.wrapping_add(1); + let mut previous_is_zero = false; + let mut out_pos = 0usize; + + while symbol < alphabet_size && remaining > 1 { + if previous_is_zero { + let mut start = symbol; + while symbol < alphabet_size && fse_read_normalized(normalized_counter, symbol) == 0 { + symbol = symbol.wrapping_add(1); + } + if symbol == alphabet_size { + break; + } + while symbol >= start.wrapping_add(24) { + start = start.wrapping_add(24); + bit_stream = bit_stream.wrapping_add(0xffffu32 << bit_count); + if let Err(error) = + fse_write_two_bytes(buffer, header_buffer_size, &mut out_pos, bit_stream) + { + return error; + } + bit_stream >>= 16; + // The 16 just-written repeat bits exactly replace the flushed + // bits, so C deliberately leaves bitCount unchanged here. + } + while symbol >= start.wrapping_add(3) { + start = start.wrapping_add(3); + bit_stream = bit_stream.wrapping_add(3u32 << bit_count); + bit_count += 2; + } + bit_stream = bit_stream.wrapping_add((symbol - start) << bit_count); + bit_count += 2; + if bit_count > 16 { + if let Err(error) = + fse_write_two_bytes(buffer, header_buffer_size, &mut out_pos, bit_stream) + { + return error; + } + bit_stream >>= 16; + bit_count -= 16; + } + } + + let mut count = fse_read_normalized(normalized_counter, symbol) as i32; + symbol = symbol.wrapping_add(1); + let max = (2 * threshold - 1) - remaining; + remaining -= count.abs(); + count += 1; + if count >= threshold { + count += max; + } + bit_stream = bit_stream.wrapping_add((count as u32) << bit_count); + bit_count += nb_bits; + bit_count -= i32::from(count < max); + previous_is_zero = count == 1; + if remaining < 1 { + return ERROR(ZstdErrorCode::Generic); + } + while remaining < threshold { + nb_bits -= 1; + threshold >>= 1; + } + + if bit_count > 16 { + if let Err(error) = + fse_write_two_bytes(buffer, header_buffer_size, &mut out_pos, bit_stream) + { + return error; + } + bit_stream >>= 16; + bit_count -= 16; + } + } + + if remaining != 1 { + return ERROR(ZstdErrorCode::Generic); + } + + if let Err(error) = fse_write_two_bytes(buffer, header_buffer_size, &mut out_pos, bit_stream) { + return error; + } + out_pos - 2 + ((bit_count + 7) as usize / 8) +} + +#[no_mangle] +pub unsafe extern "C" fn FSE_writeNCount( + buffer: *mut c_void, + buffer_size: usize, + normalized_counter: *const c_short, + max_symbol_value: c_uint, + table_log: c_uint, +) -> usize { + if table_log > FSE_MAX_TABLELOG { + return ERROR(ZstdErrorCode::TableLogTooLarge); + } + if table_log < FSE_MIN_TABLELOG { + return ERROR(ZstdErrorCode::Generic); + } + + FSE_writeNCount_generic( + buffer, + buffer_size, + normalized_counter, + max_symbol_value, + table_log, + ) +} + +#[inline] +fn fse_highbit32(value: u32) -> u32 { + 31 - value.leading_zeros() +} + +fn FSE_minTableLog(src_size: usize, max_symbol_value: u32) -> u32 { + debug_assert!(src_size > 1); + let min_bits_src = fse_highbit32(src_size as u32) + 1; + let min_bits_symbols = fse_highbit32(max_symbol_value) + 2; + min_bits_src.min(min_bits_symbols) +} + +#[no_mangle] +pub extern "C" fn FSE_optimalTableLog_internal( + max_table_log: c_uint, + src_size: usize, + max_symbol_value: c_uint, + minus: c_uint, +) -> c_uint { + let max_bits_src = fse_highbit32(src_size.wrapping_sub(1) as u32).wrapping_sub(minus); + let min_bits = FSE_minTableLog(src_size, max_symbol_value); + let mut table_log = if max_table_log == 0 { + FSE_DEFAULT_TABLELOG + } else { + max_table_log + }; + if max_bits_src < table_log { + table_log = max_bits_src; + } + if min_bits > table_log { + table_log = min_bits; + } + table_log.clamp(FSE_MIN_TABLELOG, FSE_MAX_TABLELOG) +} + +#[no_mangle] +pub extern "C" fn FSE_optimalTableLog( + max_table_log: c_uint, + src_size: usize, + max_symbol_value: c_uint, +) -> c_uint { + FSE_optimalTableLog_internal(max_table_log, src_size, max_symbol_value, 2) +} + +unsafe fn FSE_normalizeM2( + norm: *mut c_short, + table_log: u32, + count: *const c_uint, + mut total: usize, + max_symbol_value: u32, + low_prob_count: i16, +) -> usize { + const NOT_YET_ASSIGNED: i16 = -2; + let mut distributed = 0u32; + let low_threshold = (total >> table_log) as u32; + let mut low_one = ((total * 3) >> (table_log + 1)) as u32; + + for symbol in 0..=max_symbol_value { + let current_count = fse_read_count(count, symbol); + if current_count == 0 { + fse_write_normalized(norm, symbol, 0); + } else if current_count <= low_threshold { + fse_write_normalized(norm, symbol, low_prob_count); + distributed += 1; + total -= current_count as usize; + } else if current_count <= low_one { + fse_write_normalized(norm, symbol, 1); + distributed += 1; + total -= current_count as usize; + } else { + fse_write_normalized(norm, symbol, NOT_YET_ASSIGNED); + } + } + let mut to_distribute = (1u32 << table_log) - distributed; + if to_distribute == 0 { + return 0; + } + + if (total / to_distribute as usize) > low_one as usize { + low_one = ((total * 3) / (to_distribute as usize * 2)) as u32; + for symbol in 0..=max_symbol_value { + if fse_read_normalized(norm, symbol) == NOT_YET_ASSIGNED + && fse_read_count(count, symbol) <= low_one + { + fse_write_normalized(norm, symbol, 1); + distributed += 1; + total -= fse_read_count(count, symbol) as usize; + } + } + to_distribute = (1u32 << table_log) - distributed; + } + + if distributed == max_symbol_value + 1 { + let mut max_value = 0u32; + let mut max_count = 0u32; + for symbol in 0..=max_symbol_value { + let current_count = fse_read_count(count, symbol); + if current_count > max_count { + max_value = symbol; + max_count = current_count; + } + } + let value = fse_read_normalized(norm, max_value).wrapping_add(to_distribute as i16); + fse_write_normalized(norm, max_value, value); + return 0; + } + + if total == 0 { + let mut symbol = 0u32; + while to_distribute > 0 { + if fse_read_normalized(norm, symbol) > 0 { + to_distribute -= 1; + let value = fse_read_normalized(norm, symbol).wrapping_add(1); + fse_write_normalized(norm, symbol, value); + } + symbol = (symbol + 1) % (max_symbol_value + 1); + } + return 0; + } + + let v_step_log = 62 - table_log; + let mid = (1u64 << (v_step_log - 1)) - 1; + let r_step = (((1u64 << v_step_log) * to_distribute as u64) + mid) / total as u64; + let mut tmp_total = mid; + for symbol in 0..=max_symbol_value { + if fse_read_normalized(norm, symbol) == NOT_YET_ASSIGNED { + let end = tmp_total + fse_read_count(count, symbol) as u64 * r_step; + let start_weight = (tmp_total >> v_step_log) as u32; + let end_weight = (end >> v_step_log) as u32; + let weight = end_weight - start_weight; + if weight < 1 { + return ERROR(ZstdErrorCode::Generic); + } + fse_write_normalized(norm, symbol, weight as i16); + tmp_total = end; + } + } + + 0 +} + +#[no_mangle] +pub unsafe extern "C" fn FSE_normalizeCount( + normalized_counter: *mut c_short, + mut table_log: c_uint, + count: *const c_uint, + total: usize, + max_symbol_value: c_uint, + use_low_prob_count: c_uint, +) -> usize { + if table_log == 0 { + table_log = FSE_DEFAULT_TABLELOG; + } + if table_log < FSE_MIN_TABLELOG { + return ERROR(ZstdErrorCode::Generic); + } + if table_log > FSE_MAX_TABLELOG { + return ERROR(ZstdErrorCode::TableLogTooLarge); + } + if table_log < FSE_minTableLog(total, max_symbol_value) { + return ERROR(ZstdErrorCode::Generic); + } + + const RTB_TABLE: [u32; 8] = [ + 0, 473_195, 504_333, 520_860, 550_000, 700_000, 750_000, 830_000, + ]; + let low_prob_count = if use_low_prob_count != 0 { -1 } else { 1 }; + let scale = 62 - table_log; + let step = (1u64 << 62) / total as u32 as u64; + let v_step = 1u64 << (scale - 20); + let mut still_to_distribute = 1i32 << table_log; + let mut largest = 0u32; + let mut largest_probability = 0i16; + let low_threshold = (total >> table_log) as u32; + + for symbol in 0..=max_symbol_value { + let current_count = fse_read_count(count, symbol); + if current_count as usize == total { + return 0; + } + if current_count == 0 { + fse_write_normalized(normalized_counter, symbol, 0); + } else if current_count <= low_threshold { + fse_write_normalized(normalized_counter, symbol, low_prob_count); + still_to_distribute -= 1; + } else { + let mut probability = ((current_count as u64 * step) >> scale) as i16; + if probability < 8 { + let rest_to_beat = v_step * RTB_TABLE[probability as usize] as u64; + if current_count as u64 * step - ((probability as u64) << scale) > rest_to_beat { + probability += 1; + } + } + if probability > largest_probability { + largest_probability = probability; + largest = symbol; + } + fse_write_normalized(normalized_counter, symbol, probability); + still_to_distribute -= probability as i32; + } + } + + if -still_to_distribute >= (fse_read_normalized(normalized_counter, largest) >> 1) as i32 { + let error_code = FSE_normalizeM2( + normalized_counter, + table_log, + count, + total, + max_symbol_value, + low_prob_count, + ); + if ERR_isError(error_code) { + return error_code; + } + } else { + let value = fse_read_normalized(normalized_counter, largest) + .wrapping_add(still_to_distribute as i16); + fse_write_normalized(normalized_counter, largest, value); + } + + table_log as usize +} + +#[no_mangle] +pub unsafe extern "C" fn FSE_buildCTable_rle(ct: *mut c_uint, symbol_value: u8) -> usize { + let ctable = ct as *mut u8; + fse_write_u16(ctable, 0, 0); + fse_write_u16(ctable, 1, symbol_value as u16); + fse_write_u16(ctable, 2, 0); + fse_write_u16(ctable, 3, 0); + fse_write_transform( + ct, + 0, + symbol_value as usize, + FSE_symbolCompressionTransform { + deltaFindState: 0, + deltaNbBits: 0, + }, + ); + 0 +} + +unsafe fn FSE_initCState(state: *mut FSE_CState_t, ct: *const c_uint) { + let ctable = ct as *const u8; + let table_log = fse_read_u16(ctable, 0) as u32; + *state = FSE_CState_t { + value: (1usize << table_log) as isize, + stateTable: ctable.add(4) as *const c_void, + symbolTT: ctable.add(fse_ctable_transform_offset(table_log)) as *const c_void, + stateLog: table_log, + }; +} + +unsafe fn FSE_initCState2(state: *mut FSE_CState_t, ct: *const c_uint, symbol: u32) { + FSE_initCState(state, ct); + let transform = fse_read_transform(ct, (*state).stateLog, symbol as usize); + let nb_bits_out = transform.deltaNbBits.wrapping_add(1 << 15) >> 16; + let value = (nb_bits_out << 16).wrapping_sub(transform.deltaNbBits); + (*state).value = value as isize; + let index = ((*state).value >> nb_bits_out) + transform.deltaFindState as isize; + (*state).value = ((*state).stateTable as *const u16) + .add(index as usize) + .read_unaligned() as isize; +} + +unsafe fn FSE_encodeSymbol(bit_stream: *mut BIT_CStream_t, state: *mut FSE_CState_t, symbol: u32) { + let transform = ((*state).symbolTT as *const FSE_symbolCompressionTransform) + .add(symbol as usize) + .read_unaligned(); + let nb_bits_out = (((*state).value as u64 + transform.deltaNbBits as u64) >> 16) as u32; + BIT_addBits(bit_stream, (*state).value as usize, nb_bits_out); + let index = ((*state).value >> nb_bits_out) + transform.deltaFindState as isize; + (*state).value = ((*state).stateTable as *const u16) + .add(index as usize) + .read_unaligned() as isize; +} + +unsafe fn FSE_flushCState(bit_stream: *mut BIT_CStream_t, state: *const FSE_CState_t) { + BIT_addBits(bit_stream, (*state).value as usize, (*state).stateLog); + BIT_flushBits(bit_stream); +} + +unsafe fn FSE_compress_usingCTable_generic( + dst: *mut c_void, + dst_size: usize, + src: *const c_void, + mut src_size: usize, + ct: *const c_uint, + fast: bool, +) -> usize { + if src_size <= 2 { + return 0; + } + + let mut bit_stream = std::mem::zeroed::(); + if ERR_isError(BIT_initCStream(&mut bit_stream, dst, dst_size)) { + return 0; + } + + let start = src as *const u8; + let mut input = start.add(src_size); + let mut state1 = std::mem::zeroed::(); + let mut state2 = std::mem::zeroed::(); + + if src_size & 1 != 0 { + input = input.sub(1); + FSE_initCState2(&mut state1, ct, *input as u32); + input = input.sub(1); + FSE_initCState2(&mut state2, ct, *input as u32); + input = input.sub(1); + FSE_encodeSymbol(&mut bit_stream, &mut state1, *input as u32); + if fast { + BIT_flushBitsFast(&mut bit_stream); + } else { + BIT_flushBits(&mut bit_stream); + } + } else { + input = input.sub(1); + FSE_initCState2(&mut state2, ct, *input as u32); + input = input.sub(1); + FSE_initCState2(&mut state1, ct, *input as u32); + } + + src_size -= 2; + if usize::BITS > FSE_MAX_TABLELOG * 4 + 7 && src_size & 2 != 0 { + input = input.sub(1); + FSE_encodeSymbol(&mut bit_stream, &mut state2, *input as u32); + input = input.sub(1); + FSE_encodeSymbol(&mut bit_stream, &mut state1, *input as u32); + if fast { + BIT_flushBitsFast(&mut bit_stream); + } else { + BIT_flushBits(&mut bit_stream); + } + } + + while input > start { + input = input.sub(1); + FSE_encodeSymbol(&mut bit_stream, &mut state2, *input as u32); + if usize::BITS < FSE_MAX_TABLELOG * 2 + 7 { + if fast { + BIT_flushBitsFast(&mut bit_stream); + } else { + BIT_flushBits(&mut bit_stream); + } + } + + input = input.sub(1); + FSE_encodeSymbol(&mut bit_stream, &mut state1, *input as u32); + if usize::BITS > FSE_MAX_TABLELOG * 4 + 7 { + input = input.sub(1); + FSE_encodeSymbol(&mut bit_stream, &mut state2, *input as u32); + input = input.sub(1); + FSE_encodeSymbol(&mut bit_stream, &mut state1, *input as u32); + } + if fast { + BIT_flushBitsFast(&mut bit_stream); + } else { + BIT_flushBits(&mut bit_stream); + } + } + + FSE_flushCState(&mut bit_stream, &state2); + FSE_flushCState(&mut bit_stream, &state1); + BIT_closeCStream(&mut bit_stream) +} + +#[no_mangle] +pub unsafe extern "C" fn FSE_compress_usingCTable( + dst: *mut c_void, + dst_size: usize, + src: *const c_void, + src_size: usize, + ct: *const c_uint, +) -> usize { + let block_bound = src_size + .wrapping_add(src_size >> 7) + .wrapping_add(4) + .wrapping_add(size_of::()); + FSE_compress_usingCTable_generic(dst, dst_size, src, src_size, ct, dst_size >= block_bound) +} + +#[no_mangle] +pub extern "C" fn FSE_compressBound(size: usize) -> usize { + FSE_NCOUNTBOUND + .wrapping_add(size) + .wrapping_add(size >> 7) + .wrapping_add(4) + .wrapping_add(size_of::()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::fse_decompress::FSE_decompress_wksp_bmi2; + use std::slice; + + const NO_LOW_NORM: [i16; 5] = [5, 4, 7, 6, 10]; + const NO_LOW_SOURCE: [u8; 29] = [ + 0, 1, 2, 3, 4, 3, 2, 1, 0, 4, 2, 4, 3, 1, 0, 4, 2, 3, 4, 4, 0, 1, 2, 3, 4, 0, 2, 1, 4, + ]; + const LOW_NORM: [i16; 5] = [6, -1, 5, 8, 12]; + const LOW_SOURCE: [u8; 33] = [ + 4, 3, 2, 1, 0, 4, 4, 3, 2, 4, 0, 1, 4, 3, 2, 0, 4, 4, 3, 2, 1, 0, 4, 3, 2, 4, 0, 4, 3, 2, + 1, 4, 0, + ]; + + fn bytes(hex: &str) -> Vec { + assert_eq!(hex.len() % 2, 0); + hex.as_bytes() + .chunks_exact(2) + .map(|pair| { + fn digit(value: u8) -> u8 { + match value { + b'0'..=b'9' => value - b'0', + b'a'..=b'f' => value - b'a' + 10, + _ => panic!("invalid hex digit"), + } + } + (digit(pair[0]) << 4) | digit(pair[1]) + }) + .collect() + } + + fn ctable_storage(table_log: u32, max_symbol_value: u32) -> Vec { + vec![0; 1 + (1usize << (table_log - 1)) + (max_symbol_value as usize + 1) * 2] + } + + fn workspace(table_log: u32, max_symbol_value: u32) -> Vec { + let bytes = fse_build_ctable_workspace_size(max_symbol_value, table_log).unwrap(); + vec![0; bytes.div_ceil(4)] + } + + unsafe fn build_table(norm: &[i16], table_log: u32) -> Vec { + let mut table = ctable_storage(table_log, norm.len() as u32 - 1); + let mut wksp = workspace(table_log, norm.len() as u32 - 1); + assert_eq!( + FSE_buildCTable_wksp( + table.as_mut_ptr(), + norm.as_ptr(), + norm.len() as u32 - 1, + table_log, + wksp.as_mut_ptr().cast(), + wksp.len() * size_of::(), + ), + 0 + ); + table + } + + unsafe fn compress(norm: &[i16], source: &[u8]) -> (Vec, Vec) { + let table = build_table(norm, 5); + let mut payload = vec![0u8; FSE_compressBound(source.len())]; + let written = FSE_compress_usingCTable( + payload.as_mut_ptr().cast(), + payload.len(), + source.as_ptr().cast(), + source.len(), + table.as_ptr(), + ); + assert_ne!(written, 0); + payload.truncate(written); + + let mut header = vec![0u8; FSE_NCOUNTBOUND]; + let header_size = FSE_writeNCount( + header.as_mut_ptr().cast(), + header.len(), + norm.as_ptr(), + norm.len() as u32 - 1, + 5, + ); + assert!(!ERR_isError(header_size)); + header.truncate(header_size); + (header, payload) + } + + unsafe fn compress_with_capacity(norm: &[i16], source: &[u8], capacity: usize) -> Vec { + let table = build_table(norm, 5); + let mut payload = vec![0u8; capacity]; + let written = FSE_compress_usingCTable( + payload.as_mut_ptr().cast(), + payload.len(), + source.as_ptr().cast(), + source.len(), + table.as_ptr(), + ); + payload.truncate(written); + payload + } + + unsafe fn decompress(encoded: &[u8], capacity: usize) -> (usize, Vec) { + let mut output = vec![0u8; capacity]; + let mut workspace = vec![0u32; 6000]; + let result = FSE_decompress_wksp_bmi2( + output.as_mut_ptr().cast(), + output.len(), + encoded.as_ptr().cast(), + encoded.len(), + FSE_MAX_TABLELOG, + workspace.as_mut_ptr().cast(), + workspace.len() * size_of::(), + 0, + ); + (result, output) + } + + #[test] + fn ctable_layout_matches_pristine_c_for_both_spread_paths() { + let no_low = unsafe { build_table(&NO_LOW_NORM, 5) }; + let low = unsafe { build_table(&LOW_NORM, 5) }; + let no_low_bytes = unsafe { + slice::from_raw_parts( + no_low.as_ptr().cast::(), + no_low.len() * size_of::(), + ) + }; + let low_bytes = unsafe { + slice::from_raw_parts(low.as_ptr().cast::(), low.len() * size_of::()) + }; + + // Fixture generated by the pristine HEAD C implementation. + assert_eq!( + no_low_bytes, + bytes("05000400200025002e0037003c0021002a0033003800220026002b002f00340039003d00230027002c00300035003e002400280029002d003100320036003a003b003f00fbffffffd8ff020001000000c0ff030002000000c8ff02000a000000d0ff02000c000000d8ff0100") + ); + assert_eq!( + low_bytes, + bytes("05000400200025002e00330037003c003f00210026002a002f003800220027002b003000340039003d003e0023002400280029002c002d0031003200350036003a003b00faffffffd0ff020005000000e0ff040002000000d8ff020004000000c0ff020008000000d0ff0100") + ); + } + + #[test] + fn ncount_and_payloads_are_bit_exact_against_pristine_c() { + let (no_low_header, no_low_payload) = unsafe { compress(&NO_LOW_NORM, &NO_LOW_SOURCE) }; + let (low_header, low_payload) = unsafe { compress(&LOW_NORM, &LOW_SOURCE) }; + assert_eq!(no_low_header, bytes("600aba07")); + assert_eq!(no_low_payload, bytes("f4d9b22b7166ab4b7001")); + assert_eq!(low_header, bytes("70c0e403")); + assert_eq!(low_payload, bytes("e0fb60f4a95030d396fb1d")); + } + + #[test] + fn bounded_ctable_stream_path_matches_the_fast_stream() { + for (norm, source, expected) in [ + ( + &NO_LOW_NORM[..], + &NO_LOW_SOURCE[..], + bytes("f4d9b22b7166ab4b7001"), + ), + ( + &LOW_NORM[..], + &LOW_SOURCE[..], + bytes("e0fb60f4a95030d396fb1d"), + ), + ] { + let capacity = source.len() + (source.len() >> 7) + 4 + size_of::() - 1; + assert_eq!( + unsafe { compress_with_capacity(norm, source, capacity) }, + expected + ); + assert!(unsafe { compress_with_capacity(norm, source, 8) }.is_empty()); + } + } + + #[test] + fn fse_streams_round_trip_through_the_rust_decoder() { + for (norm, source) in [ + (&NO_LOW_NORM[..], &NO_LOW_SOURCE[..]), + (&LOW_NORM[..], &LOW_SOURCE[..]), + ] { + let (header, payload) = unsafe { compress(norm, source) }; + let mut encoded = header; + encoded.extend_from_slice(&payload); + let (result, decoded) = unsafe { decompress(&encoded, source.len()) }; + assert_eq!(result, source.len()); + assert_eq!(&decoded[..result], source); + } + } + + #[test] + fn normalization_and_public_bounds_match_pristine_c() { + let count = [3u32, 1, 4, 1, 5, 9, 2, 6]; + let mut normalized = [0i16; 8]; + let result = + unsafe { FSE_normalizeCount(normalized.as_mut_ptr(), 5, count.as_ptr(), 31, 7, 1) }; + assert_eq!(result, 5); + assert_eq!(normalized, [3, 1, 4, 1, 5, 10, 2, 6]); + assert_eq!(FSE_NCountWriteBound(4, 5), 6); + assert_eq!(FSE_compressBound(33), 557); + assert_eq!(FSE_optimalTableLog(12, 33, 4), 5); + } + + #[test] + fn rle_ctable_uses_the_c_layout() { + let mut table = vec![0xa5a5_a5a5u32; 1 + 1 + (256 * 2)]; + assert_eq!(unsafe { FSE_buildCTable_rle(table.as_mut_ptr(), 127) }, 0); + let bytes = unsafe { slice::from_raw_parts(table.as_ptr().cast::(), table.len() * 4) }; + assert_eq!(&bytes[..8], &[0, 0, 127, 0, 0, 0, 0, 0]); + let transform_offset = + fse_ctable_transform_offset(0) + 127 * size_of::(); + assert_eq!(&bytes[transform_offset..transform_offset + 8], &[0; 8]); + } + + #[test] + fn compact_buffer_path_does_not_overwrite_its_boundary() { + let count = [1i16, 0, 1, 0, 1, 0, 1, 0, 1, 9, 18]; + let mut storage = [0xa5u8; 11]; + let result = + unsafe { FSE_writeNCount(storage.as_mut_ptr().cast(), 9, count.as_ptr(), 10, 5) }; + assert!(ERR_isError(result) || result <= 9); + assert_eq!(&storage[9..], &[0xa5, 0xa5]); + } + + #[test] + fn generated_normalized_counts_round_trip() { + let mut state = 0x7a4d_3e21u32; + for case_index in 0..128u32 { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + let table_log = 5 + state % 4; + let max_symbol_value = 3 + (state >> 8) % 36; + let mut normalized = vec![0i16; max_symbol_value as usize + 1]; + let mut remaining = 1u32 << table_log; + for symbol in 0..max_symbol_value { + state = state + .wrapping_mul(1_664_525) + .wrapping_add(1_013_904_223 ^ case_index); + // Reserve one unit for the final symbol. This is the valid + // normalized-count contract expected by FSE_writeNCount(). + let weight = state % remaining; + normalized[symbol as usize] = if weight == 1 && state & 1 != 0 { + -1 + } else { + weight as i16 + }; + remaining -= weight; + } + normalized[max_symbol_value as usize] = if remaining == 1 && state & 2 != 0 { + -1 + } else { + remaining as i16 + }; + + let mut header = [0u8; FSE_NCOUNTBOUND]; + let written = unsafe { + FSE_writeNCount( + header.as_mut_ptr().cast(), + header.len(), + normalized.as_ptr(), + max_symbol_value, + table_log, + ) + }; + assert!( + !ERR_isError(written), + "case {case_index}, table_log {table_log}, normalized {normalized:?}, written {written}" + ); + let mut decoded = vec![0i16; 256]; + let mut decoded_max = 255u32; + let mut decoded_log = 0u32; + let read = unsafe { + crate::entropy_common::FSE_readNCount( + decoded.as_mut_ptr(), + &mut decoded_max, + &mut decoded_log, + header.as_ptr().cast(), + written, + ) + }; + assert_eq!(read, written, "case {case_index}"); + assert_eq!(decoded_max, max_symbol_value, "case {case_index}"); + assert_eq!(decoded_log, table_log, "case {case_index}"); + assert_eq!( + &decoded[..normalized.len()], + normalized.as_slice(), + "case {case_index}" + ); + } + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 7990c0f2d..ae2e63e28 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -7,6 +7,7 @@ pub mod cpu; pub mod debug; pub mod entropy_common; pub mod errors; +pub mod fse_compress; pub mod fse_decompress; pub mod hist; pub mod mem;