diff --git a/lib/compress/zstd_compress_sequences.c b/lib/compress/zstd_compress_sequences.c index 7beb9daa6..b222905bf 100644 --- a/lib/compress/zstd_compress_sequences.c +++ b/lib/compress/zstd_compress_sequences.c @@ -8,435 +8,10 @@ * You may select, at your option, one of the above-listed licenses. */ - /*-************************************* - * Dependencies - ***************************************/ +/* + * Sequence entropy coding is implemented in + * rust/src/zstd_compress_sequences.rs. Keep this declaration-compatible + * translation unit in the native source list so the C callers retain their + * existing internal header configuration while linking the Rust ABI exports. + */ #include "zstd_compress_sequences.h" - -/** - * -log2(x / 256) lookup table for x in [0, 256). - * If x == 0: Return 0 - * Else: Return floor(-log2(x / 256) * 256) - */ -static unsigned const kInverseProbabilityLog256[256] = { - 0, 2048, 1792, 1642, 1536, 1453, 1386, 1329, 1280, 1236, 1197, 1162, - 1130, 1100, 1073, 1047, 1024, 1001, 980, 960, 941, 923, 906, 889, - 874, 859, 844, 830, 817, 804, 791, 779, 768, 756, 745, 734, - 724, 714, 704, 694, 685, 676, 667, 658, 650, 642, 633, 626, - 618, 610, 603, 595, 588, 581, 574, 567, 561, 554, 548, 542, - 535, 529, 523, 517, 512, 506, 500, 495, 489, 484, 478, 473, - 468, 463, 458, 453, 448, 443, 438, 434, 429, 424, 420, 415, - 411, 407, 402, 398, 394, 390, 386, 382, 377, 373, 370, 366, - 362, 358, 354, 350, 347, 343, 339, 336, 332, 329, 325, 322, - 318, 315, 311, 308, 305, 302, 298, 295, 292, 289, 286, 282, - 279, 276, 273, 270, 267, 264, 261, 258, 256, 253, 250, 247, - 244, 241, 239, 236, 233, 230, 228, 225, 222, 220, 217, 215, - 212, 209, 207, 204, 202, 199, 197, 194, 192, 190, 187, 185, - 182, 180, 178, 175, 173, 171, 168, 166, 164, 162, 159, 157, - 155, 153, 151, 149, 146, 144, 142, 140, 138, 136, 134, 132, - 130, 128, 126, 123, 121, 119, 117, 115, 114, 112, 110, 108, - 106, 104, 102, 100, 98, 96, 94, 93, 91, 89, 87, 85, - 83, 82, 80, 78, 76, 74, 73, 71, 69, 67, 66, 64, - 62, 61, 59, 57, 55, 54, 52, 50, 49, 47, 46, 44, - 42, 41, 39, 37, 36, 34, 33, 31, 30, 28, 26, 25, - 23, 22, 20, 19, 17, 16, 14, 13, 11, 10, 8, 7, - 5, 4, 2, 1, -}; - -static unsigned ZSTD_getFSEMaxSymbolValue(FSE_CTable const* ctable) { - void const* ptr = ctable; - U16 const* u16ptr = (U16 const*)ptr; - U32 const maxSymbolValue = MEM_read16(u16ptr + 1); - return maxSymbolValue; -} - -/** - * Returns true if we should use ncount=-1 else we should - * use ncount=1 for low probability symbols instead. - */ -static unsigned ZSTD_useLowProbCount(size_t const nbSeq) -{ - /* Heuristic: This should cover most blocks <= 16K and - * start to fade out after 16K to about 32K depending on - * compressibility. - */ - return nbSeq >= 2048; -} - -/** - * Returns the cost in bytes of encoding the normalized count header. - * Returns an error if any of the helper functions return an error. - */ -static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max, - size_t const nbSeq, unsigned const FSELog) -{ - BYTE wksp[FSE_NCOUNTBOUND]; - S16 norm[MaxSeq + 1]; - const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); - FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max, ZSTD_useLowProbCount(nbSeq)), ""); - return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog); -} - -/** - * Returns the cost in bits of encoding the distribution described by count - * using the entropy bound. - */ -static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t const total) -{ - unsigned cost = 0; - unsigned s; - - assert(total > 0); - for (s = 0; s <= max; ++s) { - unsigned norm = (unsigned)((256 * count[s]) / total); - if (count[s] != 0 && norm == 0) - norm = 1; - assert(count[s] < total); - cost += count[s] * kInverseProbabilityLog256[norm]; - } - return cost >> 8; -} - -/** - * Returns the cost in bits of encoding the distribution in count using ctable. - * Returns an error if ctable cannot represent all the symbols in count. - */ -size_t ZSTD_fseBitCost( - FSE_CTable const* ctable, - unsigned const* count, - unsigned const max) -{ - unsigned const kAccuracyLog = 8; - size_t cost = 0; - unsigned s; - FSE_CState_t cstate; - FSE_initCState(&cstate, ctable); - if (ZSTD_getFSEMaxSymbolValue(ctable) < max) { - DEBUGLOG(5, "Repeat FSE_CTable has maxSymbolValue %u < %u", - ZSTD_getFSEMaxSymbolValue(ctable), max); - return ERROR(GENERIC); - } - for (s = 0; s <= max; ++s) { - unsigned const tableLog = cstate.stateLog; - unsigned const badCost = (tableLog + 1) << kAccuracyLog; - unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog); - if (count[s] == 0) - continue; - if (bitCost >= badCost) { - DEBUGLOG(5, "Repeat FSE_CTable has Prob[%u] == 0", s); - return ERROR(GENERIC); - } - cost += (size_t)count[s] * bitCost; - } - return cost >> kAccuracyLog; -} - -/** - * Returns the cost in bits of encoding the distribution in count using the - * table described by norm. The max symbol support by norm is assumed >= max. - * norm must be valid for every symbol with non-zero probability in count. - */ -size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, - unsigned const* count, unsigned const max) -{ - unsigned const shift = 8 - accuracyLog; - size_t cost = 0; - unsigned s; - assert(accuracyLog <= 8); - for (s = 0; s <= max; ++s) { - unsigned const normAcc = (norm[s] != -1) ? (unsigned)norm[s] : 1; - unsigned const norm256 = normAcc << shift; - assert(norm256 > 0); - assert(norm256 < 256); - cost += count[s] * kInverseProbabilityLog256[norm256]; - } - return cost >> 8; -} - -SymbolEncodingType_e -ZSTD_selectEncodingType( - FSE_repeat* repeatMode, unsigned const* count, unsigned const max, - size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, - FSE_CTable const* prevCTable, - short const* defaultNorm, U32 defaultNormLog, - ZSTD_DefaultPolicy_e const isDefaultAllowed, - ZSTD_strategy const strategy) -{ - ZSTD_STATIC_ASSERT(ZSTD_defaultDisallowed == 0 && ZSTD_defaultAllowed != 0); - if (mostFrequent == nbSeq) { - *repeatMode = FSE_repeat_none; - if (isDefaultAllowed && nbSeq <= 2) { - /* Prefer set_basic over set_rle when there are 2 or fewer symbols, - * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol. - * If basic encoding isn't possible, always choose RLE. - */ - DEBUGLOG(5, "Selected set_basic"); - return set_basic; - } - DEBUGLOG(5, "Selected set_rle"); - return set_rle; - } - if (strategy < ZSTD_lazy) { - if (isDefaultAllowed) { - size_t const staticFse_nbSeq_max = 1000; - size_t const mult = 10 - strategy; - size_t const baseLog = 3; - size_t const dynamicFse_nbSeq_min = (((size_t)1 << defaultNormLog) * mult) >> baseLog; /* 28-36 for offset, 56-72 for lengths */ - assert(defaultNormLog >= 5 && defaultNormLog <= 6); /* xx_DEFAULTNORMLOG */ - assert(mult <= 9 && mult >= 7); - if ( (*repeatMode == FSE_repeat_valid) - && (nbSeq < staticFse_nbSeq_max) ) { - DEBUGLOG(5, "Selected set_repeat"); - return set_repeat; - } - if ( (nbSeq < dynamicFse_nbSeq_min) - || (mostFrequent < (nbSeq >> (defaultNormLog-1))) ) { - DEBUGLOG(5, "Selected set_basic"); - /* The format allows default tables to be repeated, but it isn't useful. - * When using simple heuristics to select encoding type, we don't want - * to confuse these tables with dictionaries. When running more careful - * analysis, we don't need to waste time checking both repeating tables - * and default tables. - */ - *repeatMode = FSE_repeat_none; - return set_basic; - } - } - } else { - size_t const basicCost = isDefaultAllowed ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, count, max) : ERROR(GENERIC); - size_t const repeatCost = *repeatMode != FSE_repeat_none ? ZSTD_fseBitCost(prevCTable, count, max) : ERROR(GENERIC); - size_t const NCountCost = ZSTD_NCountCost(count, max, nbSeq, FSELog); - size_t const compressedCost = (NCountCost << 3) + ZSTD_entropyCost(count, max, nbSeq); - - if (isDefaultAllowed) { - assert(!ZSTD_isError(basicCost)); - assert(!(*repeatMode == FSE_repeat_valid && ZSTD_isError(repeatCost))); - } - assert(!ZSTD_isError(NCountCost)); - assert(compressedCost < ERROR(maxCode)); - DEBUGLOG(5, "Estimated bit costs: basic=%u\trepeat=%u\tcompressed=%u", - (unsigned)basicCost, (unsigned)repeatCost, (unsigned)compressedCost); - if (basicCost <= repeatCost && basicCost <= compressedCost) { - DEBUGLOG(5, "Selected set_basic"); - assert(isDefaultAllowed); - *repeatMode = FSE_repeat_none; - return set_basic; - } - if (repeatCost <= compressedCost) { - DEBUGLOG(5, "Selected set_repeat"); - assert(!ZSTD_isError(repeatCost)); - return set_repeat; - } - assert(compressedCost < basicCost && compressedCost < repeatCost); - } - DEBUGLOG(5, "Selected set_compressed"); - *repeatMode = FSE_repeat_check; - return set_compressed; -} - -typedef struct { - S16 norm[MaxSeq + 1]; - U32 wksp[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(MaxSeq, MaxFSELog)]; -} ZSTD_BuildCTableWksp; - -size_t -ZSTD_buildCTable(void* dst, size_t dstCapacity, - FSE_CTable* nextCTable, U32 FSELog, SymbolEncodingType_e type, - unsigned* count, U32 max, - const BYTE* codeTable, size_t nbSeq, - const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, - const FSE_CTable* prevCTable, size_t prevCTableSize, - void* entropyWorkspace, size_t entropyWorkspaceSize) -{ - BYTE* op = (BYTE*)dst; - const BYTE* const oend = op + dstCapacity; - DEBUGLOG(6, "ZSTD_buildCTable (dstCapacity=%u)", (unsigned)dstCapacity); - - switch (type) { - case set_rle: - FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max), ""); - RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall, "not enough space"); - *op = codeTable[0]; - return 1; - case set_repeat: - ZSTD_memcpy(nextCTable, prevCTable, prevCTableSize); - return 0; - case set_basic: - FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize), ""); /* note : could be pre-calculated */ - return 0; - case set_compressed: { - ZSTD_BuildCTableWksp* wksp = (ZSTD_BuildCTableWksp*)entropyWorkspace; - size_t nbSeq_1 = nbSeq; - const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); - if (count[codeTable[nbSeq-1]] > 1) { - count[codeTable[nbSeq-1]]--; - nbSeq_1--; - } - assert(nbSeq_1 > 1); - assert(entropyWorkspaceSize >= sizeof(ZSTD_BuildCTableWksp)); - (void)entropyWorkspaceSize; - FORWARD_IF_ERROR(FSE_normalizeCount(wksp->norm, tableLog, count, nbSeq_1, max, ZSTD_useLowProbCount(nbSeq_1)), "FSE_normalizeCount failed"); - assert(oend >= op); - { size_t const NCountSize = FSE_writeNCount(op, (size_t)(oend - op), wksp->norm, max, tableLog); /* overflow protected */ - FORWARD_IF_ERROR(NCountSize, "FSE_writeNCount failed"); - FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, wksp->norm, max, tableLog, wksp->wksp, sizeof(wksp->wksp)), "FSE_buildCTable_wksp failed"); - return NCountSize; - } - } - default: assert(0); RETURN_ERROR(GENERIC, "impossible to reach"); - } -} - -FORCE_INLINE_TEMPLATE size_t -ZSTD_encodeSequences_body( - void* dst, size_t dstCapacity, - FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, - FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, - FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - SeqDef const* sequences, size_t nbSeq, int longOffsets) -{ - BIT_CStream_t blockStream; - FSE_CState_t stateMatchLength; - FSE_CState_t stateOffsetBits; - FSE_CState_t stateLitLength; - - RETURN_ERROR_IF( - ERR_isError(BIT_initCStream(&blockStream, dst, dstCapacity)), - dstSize_tooSmall, "not enough space remaining"); - DEBUGLOG(6, "available space for bitstream : %i (dstCapacity=%u)", - (int)(blockStream.endPtr - blockStream.startPtr), - (unsigned)dstCapacity); - - /* first symbols */ - FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); - FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]); - FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]); - BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]); - if (MEM_32bits()) BIT_flushBits(&blockStream); - BIT_addBits(&blockStream, sequences[nbSeq-1].mlBase, ML_bits[mlCodeTable[nbSeq-1]]); - if (MEM_32bits()) BIT_flushBits(&blockStream); - if (longOffsets) { - U32 const ofBits = ofCodeTable[nbSeq-1]; - unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); - if (extraBits) { - BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, extraBits); - BIT_flushBits(&blockStream); - } - BIT_addBits(&blockStream, sequences[nbSeq-1].offBase >> extraBits, - ofBits - extraBits); - } else { - BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, ofCodeTable[nbSeq-1]); - } - BIT_flushBits(&blockStream); - - { size_t n; - for (n=nbSeq-2 ; n= 64-7-(LLFSELog+MLFSELog+OffFSELog))) - BIT_flushBits(&blockStream); /* (7)*/ - BIT_addBits(&blockStream, sequences[n].litLength, llBits); - if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream); - BIT_addBits(&blockStream, sequences[n].mlBase, mlBits); - if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream); - if (longOffsets) { - unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); - if (extraBits) { - BIT_addBits(&blockStream, sequences[n].offBase, extraBits); - BIT_flushBits(&blockStream); /* (7)*/ - } - BIT_addBits(&blockStream, sequences[n].offBase >> extraBits, - ofBits - extraBits); /* 31 */ - } else { - BIT_addBits(&blockStream, sequences[n].offBase, ofBits); /* 31 */ - } - BIT_flushBits(&blockStream); /* (7)*/ - DEBUGLOG(7, "remaining space : %i", (int)(blockStream.endPtr - blockStream.ptr)); - } } - - DEBUGLOG(6, "ZSTD_encodeSequences: flushing ML state with %u bits", stateMatchLength.stateLog); - FSE_flushCState(&blockStream, &stateMatchLength); - DEBUGLOG(6, "ZSTD_encodeSequences: flushing Off state with %u bits", stateOffsetBits.stateLog); - FSE_flushCState(&blockStream, &stateOffsetBits); - DEBUGLOG(6, "ZSTD_encodeSequences: flushing LL state with %u bits", stateLitLength.stateLog); - FSE_flushCState(&blockStream, &stateLitLength); - - { size_t const streamSize = BIT_closeCStream(&blockStream); - RETURN_ERROR_IF(streamSize==0, dstSize_tooSmall, "not enough space"); - return streamSize; - } -} - -static size_t -ZSTD_encodeSequences_default( - void* dst, size_t dstCapacity, - FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, - FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, - FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - SeqDef const* sequences, size_t nbSeq, int longOffsets) -{ - return ZSTD_encodeSequences_body(dst, dstCapacity, - CTable_MatchLength, mlCodeTable, - CTable_OffsetBits, ofCodeTable, - CTable_LitLength, llCodeTable, - sequences, nbSeq, longOffsets); -} - - -#if DYNAMIC_BMI2 - -static BMI2_TARGET_ATTRIBUTE size_t -ZSTD_encodeSequences_bmi2( - void* dst, size_t dstCapacity, - FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, - FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, - FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - SeqDef const* sequences, size_t nbSeq, int longOffsets) -{ - return ZSTD_encodeSequences_body(dst, dstCapacity, - CTable_MatchLength, mlCodeTable, - CTable_OffsetBits, ofCodeTable, - CTable_LitLength, llCodeTable, - sequences, nbSeq, longOffsets); -} - -#endif - -size_t ZSTD_encodeSequences( - void* dst, size_t dstCapacity, - FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, - FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, - FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - SeqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2) -{ - DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity); -#if DYNAMIC_BMI2 - if (bmi2) { - return ZSTD_encodeSequences_bmi2(dst, dstCapacity, - CTable_MatchLength, mlCodeTable, - CTable_OffsetBits, ofCodeTable, - CTable_LitLength, llCodeTable, - sequences, nbSeq, longOffsets); - } -#endif - (void)bmi2; - return ZSTD_encodeSequences_default(dst, dstCapacity, - CTable_MatchLength, mlCodeTable, - CTable_OffsetBits, ofCodeTable, - CTable_LitLength, llCodeTable, - sequences, nbSeq, longOffsets); -} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index abb1cc1c7..ba3e3f272 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -23,6 +23,8 @@ pub mod xxhash; pub mod zstd_common; #[cfg(feature = "compression")] pub mod zstd_compress_literals; +#[cfg(feature = "compression")] +pub mod zstd_compress_sequences; #[cfg(feature = "decompression")] pub mod zstd_ddict; #[cfg(feature = "compression")] diff --git a/rust/src/zstd_compress_sequences.rs b/rust/src/zstd_compress_sequences.rs new file mode 100644 index 000000000..26b23c8de --- /dev/null +++ b/rust/src/zstd_compress_sequences.rs @@ -0,0 +1,827 @@ +#![allow(non_snake_case)] + +//! FSE table selection, construction, and sequence bitstream encoding. +//! +//! This module translates `zstd_compress_sequences.c`. The surrounding +//! compressor still owns sequence collection and code generation in C, while +//! these routines consume its `SeqDef` and FSE-table ABI directly. + +use crate::bitstream::{ + BIT_CStream_t, BIT_addBits, BIT_closeCStream, BIT_flushBits, BIT_initCStream, +}; +use crate::common::{LL_BITS, MAX_FSE_LOG, MAX_SEQ, ML_BITS}; +use crate::errors::{ERR_isError, ZstdErrorCode, ERROR}; +use crate::fse_compress::{ + FSE_buildCTable_rle, FSE_buildCTable_wksp, FSE_normalizeCount, FSE_optimalTableLog, + FSE_writeNCount, FSE_NCOUNTBOUND, +}; +use crate::mem::MEM_32bits; +use std::ffi::c_void; +use std::mem::size_of; +use std::os::raw::{c_int, c_short, c_uint}; +use std::ptr; + +const FSE_REPEAT_NONE: c_int = 0; +const FSE_REPEAT_CHECK: c_int = 1; +const FSE_REPEAT_VALID: c_int = 2; + +const SET_BASIC: c_int = 0; +const SET_RLE: c_int = 1; +const SET_COMPRESSED: c_int = 2; +const SET_REPEAT: c_int = 3; + +const ZSTD_DEFAULT_DISALLOWED: c_int = 0; +const ZSTD_DEFAULT_ALLOWED: c_int = 1; +const ZSTD_LAZY: c_int = 4; + +const _: () = assert!(ZSTD_DEFAULT_DISALLOWED == 0 && ZSTD_DEFAULT_ALLOWED != 0); + +const FSE_CTABLE_WORKSPACE_U32: usize = + ((MAX_SEQ + 2 + (1usize << MAX_FSE_LOG)) / 2) + size_of::() / size_of::(); + +const INVERSE_PROBABILITY_LOG_256: [u32; 256] = [ + 0, 2048, 1792, 1642, 1536, 1453, 1386, 1329, 1280, 1236, 1197, 1162, 1130, 1100, 1073, 1047, + 1024, 1001, 980, 960, 941, 923, 906, 889, 874, 859, 844, 830, 817, 804, 791, 779, 768, 756, + 745, 734, 724, 714, 704, 694, 685, 676, 667, 658, 650, 642, 633, 626, 618, 610, 603, 595, 588, + 581, 574, 567, 561, 554, 548, 542, 535, 529, 523, 517, 512, 506, 500, 495, 489, 484, 478, 473, + 468, 463, 458, 453, 448, 443, 438, 434, 429, 424, 420, 415, 411, 407, 402, 398, 394, 390, 386, + 382, 377, 373, 370, 366, 362, 358, 354, 350, 347, 343, 339, 336, 332, 329, 325, 322, 318, 315, + 311, 308, 305, 302, 298, 295, 292, 289, 286, 282, 279, 276, 273, 270, 267, 264, 261, 258, 256, + 253, 250, 247, 244, 241, 239, 236, 233, 230, 228, 225, 222, 220, 217, 215, 212, 209, 207, 204, + 202, 199, 197, 194, 192, 190, 187, 185, 182, 180, 178, 175, 173, 171, 168, 166, 164, 162, 159, + 157, 155, 153, 151, 149, 146, 144, 142, 140, 138, 136, 134, 132, 130, 128, 126, 123, 121, 119, + 117, 115, 114, 112, 110, 108, 106, 104, 102, 100, 98, 96, 94, 93, 91, 89, 87, 85, 83, 82, 80, + 78, 76, 74, 73, 71, 69, 67, 66, 64, 62, 61, 59, 57, 55, 54, 52, 50, 49, 47, 46, 44, 42, 41, 39, + 37, 36, 34, 33, 31, 30, 28, 26, 25, 23, 22, 20, 19, 17, 16, 14, 13, 11, 10, 8, 7, 5, 4, 2, 1, +]; + +/// ABI-compatible `SeqDef` from `zstd_compress_internal.h`. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +pub struct SeqDef { + pub offBase: u32, + pub litLength: u16, + pub mlBase: u16, +} + +#[repr(C)] +struct FseSymbolCompressionTransform { + delta_find_state: c_int, + delta_nb_bits: u32, +} + +struct FseCState { + value: isize, + state_table: *const u16, + symbol_tt: *const FseSymbolCompressionTransform, + state_log: u32, +} + +/// Workspace layout used by `ZSTD_buildCTable()` for a compressed table. +/// It is deliberately C-shaped because the caller supplies the storage. +#[repr(C)] +struct ZstdBuildCTableWksp { + norm: [i16; MAX_SEQ + 1], + wksp: [u32; FSE_CTABLE_WORKSPACE_U32], +} + +#[inline] +fn ctable_transform_offset(table_log: u32) -> usize { + 4 + if table_log == 0 { + 4 + } else { + (1usize << table_log) * 2 + } +} + +#[inline] +unsafe fn ctable_read_u16(ctable: *const u32, index: usize) -> u16 { + unsafe { + ctable + .cast::() + .add(index * size_of::()) + .cast::() + .read_unaligned() + } +} + +#[inline] +unsafe fn fse_init_c_state(state: &mut FseCState, ctable: *const u32) { + let table_log = unsafe { ctable_read_u16(ctable, 0) } as u32; + *state = FseCState { + value: (1usize << table_log) as isize, + state_table: unsafe { ctable.cast::().add(4).cast::() }, + symbol_tt: unsafe { + ctable + .cast::() + .add(ctable_transform_offset(table_log)) + .cast::() + }, + state_log: table_log, + }; +} + +#[inline] +unsafe fn fse_init_c_state2(state: &mut FseCState, ctable: *const u32, symbol: u32) { + unsafe { fse_init_c_state(state, ctable) }; + let transform = unsafe { state.symbol_tt.add(symbol as usize).read_unaligned() }; + let nb_bits_out = transform.delta_nb_bits.wrapping_add(1 << 15) >> 16; + state.value = (nb_bits_out << 16).wrapping_sub(transform.delta_nb_bits) as isize; + let index = (state.value >> nb_bits_out) + transform.delta_find_state as isize; + state.value = unsafe { state.state_table.add(index as usize).read_unaligned() } as isize; +} + +#[inline] +unsafe fn fse_encode_symbol(bit_stream: *mut BIT_CStream_t, state: &mut FseCState, symbol: u32) { + let transform = unsafe { state.symbol_tt.add(symbol as usize).read_unaligned() }; + let nb_bits_out = ((state.value as u64 + transform.delta_nb_bits as u64) >> 16) as u32; + unsafe { BIT_addBits(bit_stream, state.value as usize, nb_bits_out) }; + let index = (state.value >> nb_bits_out) + transform.delta_find_state as isize; + state.value = unsafe { state.state_table.add(index as usize).read_unaligned() } as isize; +} + +#[inline] +unsafe fn fse_flush_c_state(bit_stream: *mut BIT_CStream_t, state: &FseCState) { + unsafe { + BIT_addBits(bit_stream, state.value as usize, state.state_log); + BIT_flushBits(bit_stream); + } +} + +#[inline] +fn fse_bit_cost( + transform: FseSymbolCompressionTransform, + table_log: u32, + accuracy_log: u32, +) -> u32 { + let min_nb_bits = transform.delta_nb_bits >> 16; + let threshold = (min_nb_bits + 1) << 16; + let table_size = 1u32 << table_log; + let delta_from_threshold = + threshold.wrapping_sub(transform.delta_nb_bits.wrapping_add(table_size)); + let normalized = (delta_from_threshold << accuracy_log) >> table_log; + ((min_nb_bits + 1) << accuracy_log).wrapping_sub(normalized) +} + +#[inline] +unsafe fn zstd_get_fse_max_symbol_value(ctable: *const u32) -> u32 { + unsafe { ctable_read_u16(ctable, 1) as u32 } +} + +#[inline] +fn zstd_use_low_prob_count(nb_seq: usize) -> u32 { + u32::from(nb_seq >= 2048) +} + +unsafe fn zstd_ncount_cost(count: *const u32, max: u32, nb_seq: usize, fse_log: u32) -> usize { + let mut workspace = [0u8; FSE_NCOUNTBOUND]; + let mut norm = [0i16; MAX_SEQ + 1]; + let table_log = FSE_optimalTableLog(fse_log, nb_seq, max); + let normalized = unsafe { + FSE_normalizeCount( + norm.as_mut_ptr(), + table_log, + count, + nb_seq, + max, + zstd_use_low_prob_count(nb_seq), + ) + }; + if ERR_isError(normalized) { + return normalized; + } + unsafe { + FSE_writeNCount( + workspace.as_mut_ptr().cast::(), + workspace.len(), + norm.as_ptr(), + max, + table_log, + ) + } +} + +unsafe fn zstd_entropy_cost(count: *const u32, max: u32, total: usize) -> usize { + debug_assert!(total > 0); + let mut cost = 0u32; + for symbol in 0..=max as usize { + let count_value = unsafe { *count.add(symbol) }; + let mut norm = (256usize * count_value as usize) / total; + if count_value != 0 && norm == 0 { + norm = 1; + } + debug_assert!((count_value as usize) < total); + cost = cost.wrapping_add(count_value.wrapping_mul(INVERSE_PROBABILITY_LOG_256[norm])); + } + (cost >> 8) as usize +} + +/// Estimates the FSE bit cost using an existing compression table. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_fseBitCost(ctable: *const u32, count: *const u32, max: u32) -> usize { + const ACCURACY_LOG: u32 = 8; + + let mut state = FseCState { + value: 0, + state_table: ptr::null(), + symbol_tt: ptr::null(), + state_log: 0, + }; + unsafe { fse_init_c_state(&mut state, ctable) }; + if unsafe { zstd_get_fse_max_symbol_value(ctable) } < max { + return ERROR(ZstdErrorCode::Generic); + } + + let mut cost = 0usize; + for symbol in 0..=max as usize { + let table_log = state.state_log; + let bad_cost = (table_log + 1) << ACCURACY_LOG; + let bit_cost = fse_bit_cost( + unsafe { state.symbol_tt.add(symbol).read_unaligned() }, + table_log, + ACCURACY_LOG, + ); + if unsafe { *count.add(symbol) } == 0 { + continue; + } + if bit_cost >= bad_cost { + return ERROR(ZstdErrorCode::Generic); + } + cost = cost.wrapping_add((unsafe { *count.add(symbol) }).wrapping_mul(bit_cost) as usize); + } + cost >> ACCURACY_LOG +} + +/// Estimates a normalized distribution's coding cost. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_crossEntropyCost( + norm: *const c_short, + accuracy_log: c_uint, + count: *const c_uint, + max: c_uint, +) -> usize { + debug_assert!(accuracy_log <= 8); + let shift = 8u32.saturating_sub(accuracy_log); + let mut cost = 0usize; + for symbol in 0..=max as usize { + let normalized = unsafe { *norm.add(symbol) }; + let norm_acc = if normalized != -1 { + normalized as u32 + } else { + 1 + }; + let norm_256 = (norm_acc << shift) as usize; + debug_assert!(norm_256 > 0 && norm_256 < 256); + cost = cost.wrapping_add( + (unsafe { *count.add(symbol) } as usize) + .wrapping_mul(INVERSE_PROBABILITY_LOG_256[norm_256] as usize), + ); + } + cost >> 8 +} + +/// Chooses basic, RLE, compressed, or repeated FSE table encoding. +#[allow(clippy::too_many_arguments)] +#[no_mangle] +pub unsafe extern "C" fn ZSTD_selectEncodingType( + repeat_mode: *mut c_int, + count: *const c_uint, + max: c_uint, + most_frequent: usize, + nb_seq: usize, + fse_log: c_uint, + prev_ctable: *const u32, + default_norm: *const c_short, + default_norm_log: u32, + is_default_allowed: c_int, + strategy: c_int, +) -> c_int { + if most_frequent == nb_seq { + unsafe { *repeat_mode = FSE_REPEAT_NONE }; + if is_default_allowed != 0 && nb_seq <= 2 { + return SET_BASIC; + } + return SET_RLE; + } + + if strategy < ZSTD_LAZY && is_default_allowed != 0 { + const STATIC_FSE_NBSEQ_MAX: usize = 1000; + const BASE_LOG: usize = 3; + let mult = (10 - strategy) as usize; + let dynamic_fse_nbseq_min = ((1usize << default_norm_log) * mult) >> BASE_LOG; + debug_assert!((5..=6).contains(&default_norm_log)); + debug_assert!((7..=9).contains(&mult)); + + if unsafe { *repeat_mode } == FSE_REPEAT_VALID && nb_seq < STATIC_FSE_NBSEQ_MAX { + return SET_REPEAT; + } + if nb_seq < dynamic_fse_nbseq_min || most_frequent < (nb_seq >> (default_norm_log - 1)) { + unsafe { *repeat_mode = FSE_REPEAT_NONE }; + return SET_BASIC; + } + } else if strategy >= ZSTD_LAZY { + let basic_cost = if is_default_allowed != 0 { + unsafe { ZSTD_crossEntropyCost(default_norm, default_norm_log, count, max) } + } else { + ERROR(ZstdErrorCode::Generic) + }; + let repeat_cost = if unsafe { *repeat_mode } != FSE_REPEAT_NONE { + unsafe { ZSTD_fseBitCost(prev_ctable, count, max) } + } else { + ERROR(ZstdErrorCode::Generic) + }; + let ncount_cost = unsafe { zstd_ncount_cost(count, max, nb_seq, fse_log) }; + let compressed_cost = ncount_cost + .wrapping_shl(3) + .wrapping_add(unsafe { zstd_entropy_cost(count, max, nb_seq) }); + + if basic_cost <= repeat_cost && basic_cost <= compressed_cost { + unsafe { *repeat_mode = FSE_REPEAT_NONE }; + return SET_BASIC; + } + if repeat_cost <= compressed_cost { + return SET_REPEAT; + } + } + + unsafe { *repeat_mode = FSE_REPEAT_CHECK }; + SET_COMPRESSED +} + +/// Builds an FSE compression table and, when necessary, writes its header. +#[allow(clippy::too_many_arguments)] +#[no_mangle] +pub unsafe extern "C" fn ZSTD_buildCTable( + dst: *mut c_void, + dst_capacity: usize, + next_ctable: *mut u32, + fse_log: u32, + encoding_type: c_int, + count: *mut u32, + max: u32, + code_table: *const u8, + nb_seq: usize, + default_norm: *const c_short, + default_norm_log: u32, + default_max: u32, + prev_ctable: *const u32, + prev_ctable_size: usize, + entropy_workspace: *mut c_void, + entropy_workspace_size: usize, +) -> usize { + match encoding_type { + SET_RLE => { + let result = unsafe { FSE_buildCTable_rle(next_ctable, max as u8) }; + if ERR_isError(result) { + return result; + } + if dst_capacity == 0 { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + unsafe { *dst.cast::() = *code_table }; + 1 + } + SET_REPEAT => { + unsafe { + ptr::copy_nonoverlapping( + prev_ctable.cast::(), + next_ctable.cast::(), + prev_ctable_size, + ); + } + 0 + } + SET_BASIC => unsafe { + FSE_buildCTable_wksp( + next_ctable, + default_norm, + default_max, + default_norm_log, + entropy_workspace, + entropy_workspace_size, + ) + }, + SET_COMPRESSED => { + let workspace = entropy_workspace.cast::(); + let mut nb_seq_1 = nb_seq; + let table_log = FSE_optimalTableLog(fse_log, nb_seq, max); + let last_code = unsafe { *code_table.add(nb_seq - 1) } as usize; + if unsafe { *count.add(last_code) } > 1 { + unsafe { *count.add(last_code) -= 1 }; + nb_seq_1 -= 1; + } + debug_assert!(nb_seq_1 > 1); + debug_assert!(entropy_workspace_size >= size_of::()); + + let normalized = unsafe { + FSE_normalizeCount( + ptr::addr_of_mut!((*workspace).norm).cast::(), + table_log, + count, + nb_seq_1, + max, + zstd_use_low_prob_count(nb_seq_1), + ) + }; + if ERR_isError(normalized) { + return normalized; + } + let header_size = unsafe { + FSE_writeNCount( + dst, + dst_capacity, + ptr::addr_of!((*workspace).norm).cast::(), + max, + table_log, + ) + }; + if ERR_isError(header_size) { + return header_size; + } + let result = unsafe { + FSE_buildCTable_wksp( + next_ctable, + ptr::addr_of!((*workspace).norm).cast::(), + max, + table_log, + ptr::addr_of_mut!((*workspace).wksp).cast::(), + size_of::<[u32; FSE_CTABLE_WORKSPACE_U32]>(), + ) + }; + if ERR_isError(result) { + return result; + } + header_size + } + _ => ERROR(ZstdErrorCode::Generic), + } +} + +#[allow(clippy::too_many_arguments)] +unsafe fn zstd_encode_sequences_body( + dst: *mut c_void, + dst_capacity: usize, + ctable_match_length: *const u32, + ml_code_table: *const u8, + ctable_offset_bits: *const u32, + of_code_table: *const u8, + ctable_lit_length: *const u32, + ll_code_table: *const u8, + sequences: *const SeqDef, + nb_seq: usize, + long_offsets: c_int, +) -> usize { + if nb_seq == 0 { + return ERROR(ZstdErrorCode::Generic); + } + + let mut block_stream = std::mem::zeroed::(); + if ERR_isError(unsafe { BIT_initCStream(&mut block_stream, dst, dst_capacity) }) { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + + let last = nb_seq - 1; + let last_ml_code = unsafe { *ml_code_table.add(last) }; + let last_of_code = unsafe { *of_code_table.add(last) }; + let last_ll_code = unsafe { *ll_code_table.add(last) }; + let last_sequence = unsafe { *sequences.add(last) }; + let mut state_match_length = FseCState { + value: 0, + state_table: ptr::null(), + symbol_tt: ptr::null(), + state_log: 0, + }; + let mut state_offset_bits = FseCState { + value: 0, + state_table: ptr::null(), + symbol_tt: ptr::null(), + state_log: 0, + }; + let mut state_lit_length = FseCState { + value: 0, + state_table: ptr::null(), + symbol_tt: ptr::null(), + state_log: 0, + }; + + unsafe { + fse_init_c_state2( + &mut state_match_length, + ctable_match_length, + last_ml_code as u32, + ); + fse_init_c_state2( + &mut state_offset_bits, + ctable_offset_bits, + last_of_code as u32, + ); + fse_init_c_state2( + &mut state_lit_length, + ctable_lit_length, + last_ll_code as u32, + ); + BIT_addBits( + &mut block_stream, + last_sequence.litLength as usize, + LL_BITS[last_ll_code as usize] as u32, + ); + if MEM_32bits() { + BIT_flushBits(&mut block_stream); + } + BIT_addBits( + &mut block_stream, + last_sequence.mlBase as usize, + ML_BITS[last_ml_code as usize] as u32, + ); + if MEM_32bits() { + BIT_flushBits(&mut block_stream); + } + if long_offsets != 0 { + let of_bits = last_of_code as u32; + let accumulator_min = if MEM_32bits() { 25 } else { 57 }; + let extra_bits = of_bits - of_bits.min(accumulator_min - 1); + if extra_bits != 0 { + BIT_addBits( + &mut block_stream, + last_sequence.offBase as usize, + extra_bits, + ); + BIT_flushBits(&mut block_stream); + } + BIT_addBits( + &mut block_stream, + (last_sequence.offBase >> extra_bits) as usize, + of_bits - extra_bits, + ); + } else { + BIT_addBits( + &mut block_stream, + last_sequence.offBase as usize, + last_of_code as u32, + ); + } + BIT_flushBits(&mut block_stream); + } + + for index in (0..last).rev() { + let ll_code = unsafe { *ll_code_table.add(index) }; + let of_code = unsafe { *of_code_table.add(index) }; + let ml_code = unsafe { *ml_code_table.add(index) }; + let sequence = unsafe { *sequences.add(index) }; + let ll_bits = LL_BITS[ll_code as usize] as u32; + let of_bits = of_code as u32; + let ml_bits = ML_BITS[ml_code as usize] as u32; + + unsafe { + fse_encode_symbol(&mut block_stream, &mut state_offset_bits, of_code as u32); + fse_encode_symbol(&mut block_stream, &mut state_match_length, ml_code as u32); + if MEM_32bits() { + BIT_flushBits(&mut block_stream); + } + fse_encode_symbol(&mut block_stream, &mut state_lit_length, ll_code as u32); + if MEM_32bits() + || of_bits + ml_bits + ll_bits + >= 64 + - 7 + - (crate::common::LL_FSE_LOG as u32 + + crate::common::ML_FSE_LOG as u32 + + crate::common::OFF_FSE_LOG as u32) + { + BIT_flushBits(&mut block_stream); + } + BIT_addBits(&mut block_stream, sequence.litLength as usize, ll_bits); + if MEM_32bits() && ll_bits + ml_bits > 24 { + BIT_flushBits(&mut block_stream); + } + BIT_addBits(&mut block_stream, sequence.mlBase as usize, ml_bits); + if MEM_32bits() || of_bits + ml_bits + ll_bits > 56 { + BIT_flushBits(&mut block_stream); + } + if long_offsets != 0 { + let accumulator_min = if MEM_32bits() { 25 } else { 57 }; + let extra_bits = of_bits - of_bits.min(accumulator_min - 1); + if extra_bits != 0 { + BIT_addBits(&mut block_stream, sequence.offBase as usize, extra_bits); + BIT_flushBits(&mut block_stream); + } + BIT_addBits( + &mut block_stream, + (sequence.offBase >> extra_bits) as usize, + of_bits - extra_bits, + ); + } else { + BIT_addBits(&mut block_stream, sequence.offBase as usize, of_bits); + } + BIT_flushBits(&mut block_stream); + } + } + + unsafe { + fse_flush_c_state(&mut block_stream, &state_match_length); + fse_flush_c_state(&mut block_stream, &state_offset_bits); + fse_flush_c_state(&mut block_stream, &state_lit_length); + } + let stream_size = unsafe { BIT_closeCStream(&mut block_stream) }; + if stream_size == 0 { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + stream_size +} + +/// Encodes sequence symbols and their extra bits into a reverse bitstream. +#[allow(clippy::too_many_arguments)] +#[no_mangle] +pub unsafe extern "C" fn ZSTD_encodeSequences( + dst: *mut c_void, + dst_capacity: usize, + ctable_match_length: *const u32, + ml_code_table: *const u8, + ctable_offset_bits: *const u32, + of_code_table: *const u8, + ctable_lit_length: *const u32, + ll_code_table: *const u8, + sequences: *const SeqDef, + nb_seq: usize, + long_offsets: c_int, + _bmi2: c_int, +) -> usize { + unsafe { + zstd_encode_sequences_body( + dst, + dst_capacity, + ctable_match_length, + ml_code_table, + ctable_offset_bits, + of_code_table, + ctable_lit_length, + ll_code_table, + sequences, + nb_seq, + long_offsets, + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::common::{ + LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, + OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, + }; + + fn ctable_size(max_table_log: usize, max_symbol_value: usize) -> usize { + 1 + (1 << (max_table_log - 1)) + (max_symbol_value + 1) * 2 + } + + fn ctable_workspace_words(max_symbol_value: usize, table_log: usize) -> usize { + ((max_symbol_value + 2) + (1 << table_log)) / 2 + 2 + } + + #[test] + fn abi_layouts_match_the_c_headers() { + assert_eq!(size_of::(), 8); + assert_eq!(size_of::(), 1248); + assert_eq!(size_of::(), 8); + assert_eq!(FSE_CTABLE_WORKSPACE_U32, 285); + } + + #[test] + fn cross_entropy_and_selection_follow_basic_cases() { + let count = [3u32, 1, 4, 1, 5]; + let norm = [3i16, 1, 4, 1, 7]; + assert_eq!( + unsafe { ZSTD_crossEntropyCost(norm.as_ptr(), 4, count.as_ptr(), 4) }, + 29 + ); + + let sequence_count = [2u32, 1, 1, 1]; + + let mut repeat = FSE_REPEAT_VALID; + let selection = unsafe { + ZSTD_selectEncodingType( + &mut repeat, + sequence_count.as_ptr(), + 3, + 2, + 5, + 6, + ptr::null(), + LL_DEFAULT_NORM.as_ptr(), + LL_DEFAULT_NORM_LOG, + ZSTD_DEFAULT_ALLOWED, + 1, + ) + }; + assert_eq!(selection, SET_REPEAT); + + let mut repeat = FSE_REPEAT_NONE; + let selection = unsafe { + ZSTD_selectEncodingType( + &mut repeat, + sequence_count.as_ptr(), + 3, + 2, + 5, + 6, + ptr::null(), + ML_DEFAULT_NORM.as_ptr(), + ML_DEFAULT_NORM_LOG, + ZSTD_DEFAULT_ALLOWED, + 1, + ) + }; + assert_eq!(selection, SET_BASIC); + assert_eq!(repeat, FSE_REPEAT_NONE); + } + + #[test] + fn sequence_encoder_accepts_c_layout_default_tables() { + let mut ll = vec![0u32; ctable_size(MAX_FSE_LOG, 35)]; + let mut ml = vec![0u32; ctable_size(MAX_FSE_LOG, 52)]; + let mut of = vec![0u32; ctable_size(MAX_FSE_LOG, 31)]; + let mut ll_workspace = vec![0u32; ctable_workspace_words(35, LL_DEFAULT_NORM_LOG as usize)]; + let mut ml_workspace = vec![0u32; ctable_workspace_words(52, ML_DEFAULT_NORM_LOG as usize)]; + let mut of_workspace = vec![0u32; ctable_workspace_words(28, OF_DEFAULT_NORM_LOG as usize)]; + unsafe { + assert_eq!( + FSE_buildCTable_wksp( + ll.as_mut_ptr(), + LL_DEFAULT_NORM.as_ptr(), + 35, + LL_DEFAULT_NORM_LOG, + ll_workspace.as_mut_ptr().cast(), + ll_workspace.len() * size_of::(), + ), + 0 + ); + assert_eq!( + FSE_buildCTable_wksp( + ml.as_mut_ptr(), + ML_DEFAULT_NORM.as_ptr(), + 52, + ML_DEFAULT_NORM_LOG, + ml_workspace.as_mut_ptr().cast(), + ml_workspace.len() * size_of::(), + ), + 0 + ); + assert_eq!( + FSE_buildCTable_wksp( + of.as_mut_ptr(), + OF_DEFAULT_NORM.as_ptr(), + 28, + OF_DEFAULT_NORM_LOG, + of_workspace.as_mut_ptr().cast(), + of_workspace.len() * size_of::(), + ), + 0 + ); + } + + let ll_codes = [0u8, 16, 24, 35]; + let ml_codes = [0u8, 32, 40, 52]; + let of_codes = [0u8, 1, 5, 28]; + let sequences = [ + SeqDef { + offBase: 0, + litLength: 0, + mlBase: 0, + }, + SeqDef { + offBase: 1, + litLength: 1, + mlBase: 1, + }, + SeqDef { + offBase: 31, + litLength: 15, + mlBase: 15, + }, + SeqDef { + offBase: 0x0FFF_FFFF, + litLength: u16::MAX, + mlBase: u16::MAX, + }, + ]; + let mut output = [0u8; 256]; + let size = unsafe { + ZSTD_encodeSequences( + output.as_mut_ptr().cast(), + output.len(), + ml.as_ptr(), + ml_codes.as_ptr(), + of.as_ptr(), + of_codes.as_ptr(), + ll.as_ptr(), + ll_codes.as_ptr(), + sequences.as_ptr(), + sequences.len(), + 1, + 0, + ) + }; + assert!(!ERR_isError(size)); + assert!(size > 0 && size < output.len()); + } +}