ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(rust): port block entropy statistics

Browse Source 
Continue the incremental zstd_compress.c migration with its sequence
statistics and seqStore entropy-compression layer.  The following now
live in rust/src/zstd_compress_stats.rs:

- ZSTD_seqToCodes(), exported under its original name because the C
  dictionary builder (zdict.c) and decodecorpus link against it,
- ZSTD_buildSequencesStatistics() and its dummy variant, whose result
  struct no longer crosses the language boundary,
- ZSTD_entropyCompressSeqStore_internal(), _wExtLitBuffer(), and
  ZSTD_entropyCompressSeqStore(),
- ZSTD_buildBlockEntropyStats() with its literals/sequences helpers,
- ZSTD_copyBlockSequences() and the ZSTD_updateRep() rules it shares
  with the superblock writer.

Boundary: ZSTD_CCtx and ZSTD_CCtx_params stay private to C.  These
paths read exactly two parameter fields, so the C shims keep the
original static/exported function names and forward the strategy and
ZSTD_literalsCompressionIsDisabled() as int scalars alongside the
seqStore and entropy-table leaves.  The leaf layouts (SeqDef,
SeqStore_t, ZSTD_hufCTables_t, ZSTD_fseCTables_t, entropy metadata,
SeqCollector, ZSTD_Sequence) are pinned by compile-time asserts in
zstd_compress.c and by both-pointer-width layout tests in Rust.
ZSTD_buildSeqStore, block dispatch/splitting, and the block-size
estimation helpers remain C for a later slice.

The superblock module previously round-tripped through the C export of
ZSTD_buildBlockEntropyStats and mirrored the entropy leaf structs
privately.  It now calls the crate-internal builder directly, and the
shared struct definitions moved to zstd_compress_stats; consequently
ZSTD_rust_compressSuperBlock() takes the two parameter scalars instead
of an opaque ZSTD_CCtx_params pointer, extracted by its C shim.  Its
layout and repcode tests moved with the definitions.

One C helper family gets no shim: the static
ZSTD_entropyCompressSeqStore_wExtLitBuffer() had a single caller and
was folded into the Rust implementation.

Byte-identity was verified against the pre-change compressor: COPYING,
datagen -g5000000 -s7, and datagen -g300000 -s21 -P90, each at levels
1/3/9/19 and --fast=5, plus a superblock-heavy pass at level 19 with
--target-compressed-block-size=1024; all 18 frames are byte-identical,
covering the repeat-mode state machine, longOffsets, RLE/raw fallback,
and dstSize_tooSmall paths through the block splitter and superblock.

Test plan:
- cd rust && cargo fmt --check && cargo clippy --all-targets -- -D
  warnings && cargo test --all-targets (131 tests: new reference
  vectors for seqToCodes, RLE table headers, empty-seqStore repeat
  copies, literal-stats type selection, and repcode resolution in
  copyBlockSequences)
- cargo build --release --no-default-features --features compression
  and --features decompression
- make -C tests fuzzer && ./tests/fuzzer -i1 --no-big-tests (covers
  ZSTD_generateSequences and ZDICT training over the Rust seqToCodes)
- make -C tests test-rust-lib-smoke
- ./tests/zstreamtest -i1 and ./tests/zstreamtest --newapi -t1 -i1
- ./tests/decodecorpus -t -T1s (decodecorpus links Rust seqToCodes)
- /tmp/ref vs /tmp/got frame diff as described above: byte-identical
This commit is contained in:
ddidderr
2026-07-11 23:11:01 +02:00
parent 529cd297e2
commit 039dec6c68
7 changed files with 1702 additions and 782 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+88 -587
View File
@@ -88,6 +88,67 @@ size_t ZSTD_rust_params_estimateCDictSizeFromCParams(
size_t dictSize, ZSTD_compressionParameters cParams,
int dictLoadMethod, const ZSTD_rustCDictSizing* sizing);
/* Sequence statistics and seqStore entropy compression live in Rust
* (rust/src/zstd_compress_stats.rs), which also exports ZSTD_seqToCodes()
* under its original name. The shims below extract the sequence store, the
* entropy-table leaves, and the only two ZSTD_CCtx_params fields these paths
* read (the strategy and the literals-compression switch), so the private
* parameter structure layout never crosses the language boundary. The
* shared leaf layouts are pinned by the compile-time asserts that follow. */
size_t ZSTD_rust_entropyCompressSeqStore_internal(
void* dst, size_t dstCapacity,
const void* literals, size_t litSize,
const SeqStore_t* seqStorePtr,
const ZSTD_entropyCTables_t* prevEntropy,
ZSTD_entropyCTables_t* nextEntropy,
int strategy, int disableLiteralCompression,
void* entropyWorkspace, size_t entropyWkspSize,
int bmi2);
size_t ZSTD_rust_entropyCompressSeqStore(
const SeqStore_t* seqStorePtr,
const ZSTD_entropyCTables_t* prevEntropy,
ZSTD_entropyCTables_t* nextEntropy,
int strategy, int disableLiteralCompression,
void* dst, size_t dstCapacity,
size_t srcSize,
void* entropyWorkspace, size_t entropyWkspSize,
int bmi2);
size_t ZSTD_rust_buildBlockEntropyStats(
const SeqStore_t* seqStorePtr,
const ZSTD_entropyCTables_t* prevEntropy,
ZSTD_entropyCTables_t* nextEntropy,
int strategy, int disableLiteralCompression,
ZSTD_entropyCTablesMetadata_t* entropyMetadata,
void* workspace, size_t wkspSize);
size_t ZSTD_rust_copyBlockSequences(
SeqCollector* seqCollector, const SeqStore_t* seqStore,
const U32 prevRepcodes[ZSTD_REP_NUM]);
typedef char ZSTD_rust_stats_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1];
typedef char ZSTD_rust_stats_seqstore_long_length_pos[
(offsetof(SeqStore_t, longLengthPos) == 9 * sizeof(size_t) + 4) ? 1 : -1];
typedef char ZSTD_rust_stats_seqstore_layout[
(sizeof(SeqStore_t) == 9 * sizeof(size_t) + 8) ? 1 : -1];
typedef char ZSTD_rust_stats_huf_layout[
(sizeof(ZSTD_hufCTables_t) == 258 * sizeof(size_t)) ? 1 : -1];
typedef char ZSTD_rust_stats_fse_layout[(sizeof(ZSTD_fseCTables_t) == 3552) ? 1 : -1];
typedef char ZSTD_rust_stats_entropy_fse_offset[
(offsetof(ZSTD_entropyCTables_t, fse) == sizeof(ZSTD_hufCTables_t)) ? 1 : -1];
typedef char ZSTD_rust_stats_huf_metadata_buffer[
(offsetof(ZSTD_hufCTablesMetadata_t, hufDesBuffer) == 4
&& ZSTD_MAX_HUF_HEADER_SIZE == 128) ? 1 : -1];
typedef char ZSTD_rust_stats_fse_metadata_buffer[
(offsetof(ZSTD_fseCTablesMetadata_t, fseTablesBuffer) == 12
&& ZSTD_MAX_FSE_HEADERS_SIZE == 133) ? 1 : -1];
typedef char ZSTD_rust_stats_entropy_metadata_layout[
(offsetof(ZSTD_entropyCTablesMetadata_t, fseMetadata)
== sizeof(ZSTD_hufCTablesMetadata_t)) ? 1 : -1];
typedef char ZSTD_rust_stats_sequence_layout[(sizeof(ZSTD_Sequence) == 16) ? 1 : -1];
typedef char ZSTD_rust_stats_seqcollector_layout[
(offsetof(SeqCollector, seqStart) == sizeof(size_t)
&& offsetof(SeqCollector, seqIndex) == 2 * sizeof(size_t)
&& sizeof(SeqCollector) == 4 * sizeof(size_t)) ? 1 : -1];
#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
# define ZSTD_RUST_ASAN_REDZONE_SIZE ((size_t)ZSTD_CWKSP_ASAN_REDZONE_SIZE)
#else
@@ -2561,33 +2622,7 @@ static void ZSTD_reduceIndex (ZSTD_MatchState_t* ms, ZSTD_CCtx_params const* par
/* See doc/zstd_compression_format.md for detailed format description */
int ZSTD_seqToCodes(const SeqStore_t* seqStorePtr)
{
const SeqDef* const sequences = seqStorePtr->sequencesStart;
BYTE* const llCodeTable = seqStorePtr->llCode;
BYTE* const ofCodeTable = seqStorePtr->ofCode;
BYTE* const mlCodeTable = seqStorePtr->mlCode;
U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
U32 u;
int longOffsets = 0;
assert(nbSeq <= seqStorePtr->maxNbSeq);
for (u=0; u<nbSeq; u++) {
U32 const llv = sequences[u].litLength;
U32 const ofCode = ZSTD_highbit32(sequences[u].offBase);
U32 const mlv = sequences[u].mlBase;
llCodeTable[u] = (BYTE)ZSTD_LLcode(llv);
ofCodeTable[u] = (BYTE)ofCode;
mlCodeTable[u] = (BYTE)ZSTD_MLcode(mlv);
assert(!(MEM_64bits() && ofCode >= STREAM_ACCUMULATOR_MIN));
if (MEM_32bits() && ofCode >= STREAM_ACCUMULATOR_MIN)
longOffsets = 1;
}
if (seqStorePtr->longLengthType==ZSTD_llt_literalLength)
llCodeTable[seqStorePtr->longLengthPos] = MaxLL;
if (seqStorePtr->longLengthType==ZSTD_llt_matchLength)
mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
return longOffsets;
}
/* ZSTD_seqToCodes() lives in rust/src/zstd_compress_stats.rs. */
/* ZSTD_useTargetCBlockSize():
* Returns if target compressed block size param is being used.
@@ -2611,150 +2646,11 @@ static int ZSTD_blockSplitterEnabled(ZSTD_CCtx_params* cctxParams)
return (cctxParams->postBlockSplitter == ZSTD_ps_enable);
}
/* Type returned by ZSTD_buildSequencesStatistics containing finalized symbol encoding types
* and size of the sequences statistics
*/
typedef struct {
U32 LLtype;
U32 Offtype;
U32 MLtype;
size_t size;
size_t lastCountSize; /* Accounts for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */
int longOffsets;
} ZSTD_symbolEncodingTypeStats_t;
/* ZSTD_buildSequencesStatistics() and the ZSTD_entropyCompressSeqStore*()
* implementations live in rust/src/zstd_compress_stats.rs. The wrappers
* below keep the original signatures and extract the only two
* ZSTD_CCtx_params fields those paths read. */
/* ZSTD_buildSequencesStatistics():
* Returns a ZSTD_symbolEncodingTypeStats_t, or a zstd error code in the `size` field.
* Modifies `nextEntropy` to have the appropriate values as a side effect.
* nbSeq must be greater than 0.
*
* entropyWkspSize must be of size at least ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1)*sizeof(U32)
*/
static ZSTD_symbolEncodingTypeStats_t
ZSTD_buildSequencesStatistics(
const SeqStore_t* seqStorePtr, size_t nbSeq,
const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy,
BYTE* dst, const BYTE* const dstEnd,
ZSTD_strategy strategy, unsigned* countWorkspace,
void* entropyWorkspace, size_t entropyWkspSize)
{
BYTE* const ostart = dst;
const BYTE* const oend = dstEnd;
BYTE* op = ostart;
FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable;
FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable;
FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable;
const BYTE* const ofCodeTable = seqStorePtr->ofCode;
const BYTE* const llCodeTable = seqStorePtr->llCode;
const BYTE* const mlCodeTable = seqStorePtr->mlCode;
ZSTD_symbolEncodingTypeStats_t stats;
stats.lastCountSize = 0;
/* convert length/distances into codes */
stats.longOffsets = ZSTD_seqToCodes(seqStorePtr);
assert(op <= oend);
assert(nbSeq != 0); /* ZSTD_selectEncodingType() divides by nbSeq */
/* build CTable for Literal Lengths */
{ unsigned max = MaxLL;
size_t const mostFrequent = HIST_countFast_wksp(countWorkspace, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */
DEBUGLOG(5, "Building LL table");
nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode;
stats.LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode,
countWorkspace, max, mostFrequent, nbSeq,
LLFSELog, prevEntropy->litlengthCTable,
LL_defaultNorm, LL_defaultNormLog,
ZSTD_defaultAllowed, strategy);
assert(set_basic < set_compressed && set_rle < set_compressed);
assert(!(stats.LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(
op, (size_t)(oend - op),
CTable_LitLength, LLFSELog, (SymbolEncodingType_e)stats.LLtype,
countWorkspace, max, llCodeTable, nbSeq,
LL_defaultNorm, LL_defaultNormLog, MaxLL,
prevEntropy->litlengthCTable,
sizeof(prevEntropy->litlengthCTable),
entropyWorkspace, entropyWkspSize);
if (ZSTD_isError(countSize)) {
DEBUGLOG(3, "ZSTD_buildCTable for LitLens failed");
stats.size = countSize;
return stats;
}
if (stats.LLtype == set_compressed)
stats.lastCountSize = countSize;
op += countSize;
assert(op <= oend);
} }
/* build CTable for Offsets */
{ unsigned max = MaxOff;
size_t const mostFrequent = HIST_countFast_wksp(
countWorkspace, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */
/* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */
ZSTD_DefaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed;
DEBUGLOG(5, "Building OF table");
nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode;
stats.Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode,
countWorkspace, max, mostFrequent, nbSeq,
OffFSELog, prevEntropy->offcodeCTable,
OF_defaultNorm, OF_defaultNormLog,
defaultPolicy, strategy);
assert(!(stats.Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(
op, (size_t)(oend - op),
CTable_OffsetBits, OffFSELog, (SymbolEncodingType_e)stats.Offtype,
countWorkspace, max, ofCodeTable, nbSeq,
OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
prevEntropy->offcodeCTable,
sizeof(prevEntropy->offcodeCTable),
entropyWorkspace, entropyWkspSize);
if (ZSTD_isError(countSize)) {
DEBUGLOG(3, "ZSTD_buildCTable for Offsets failed");
stats.size = countSize;
return stats;
}
if (stats.Offtype == set_compressed)
stats.lastCountSize = countSize;
op += countSize;
assert(op <= oend);
} }
/* build CTable for MatchLengths */
{ unsigned max = MaxML;
size_t const mostFrequent = HIST_countFast_wksp(
countWorkspace, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */
DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op));
nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode;
stats.MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode,
countWorkspace, max, mostFrequent, nbSeq,
MLFSELog, prevEntropy->matchlengthCTable,
ML_defaultNorm, ML_defaultNormLog,
ZSTD_defaultAllowed, strategy);
assert(!(stats.MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(
op, (size_t)(oend - op),
CTable_MatchLength, MLFSELog, (SymbolEncodingType_e)stats.MLtype,
countWorkspace, max, mlCodeTable, nbSeq,
ML_defaultNorm, ML_defaultNormLog, MaxML,
prevEntropy->matchlengthCTable,
sizeof(prevEntropy->matchlengthCTable),
entropyWorkspace, entropyWkspSize);
if (ZSTD_isError(countSize)) {
DEBUGLOG(3, "ZSTD_buildCTable for MatchLengths failed");
stats.size = countSize;
return stats;
}
if (stats.MLtype == set_compressed)
stats.lastCountSize = countSize;
op += countSize;
assert(op <= oend);
} }
stats.size = (size_t)(op-ostart);
return stats;
}
/* ZSTD_entropyCompressSeqStore_internal():
* compresses both literals and sequences
* Returns compressed size of block, or a zstd error.
*/
#define SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO 20
MEM_STATIC size_t
ZSTD_entropyCompressSeqStore_internal(
void* dst, size_t dstCapacity,
@@ -2766,150 +2662,13 @@ ZSTD_entropyCompressSeqStore_internal(
void* entropyWorkspace, size_t entropyWkspSize,
const int bmi2)
{
ZSTD_strategy const strategy = cctxParams->cParams.strategy;
unsigned* count = (unsigned*)entropyWorkspace;
FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable;
FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable;
FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable;
const SeqDef* const sequences = seqStorePtr->sequencesStart;
const size_t nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
const BYTE* const ofCodeTable = seqStorePtr->ofCode;
const BYTE* const llCodeTable = seqStorePtr->llCode;
const BYTE* const mlCodeTable = seqStorePtr->mlCode;
BYTE* const ostart = (BYTE*)dst;
BYTE* const oend = ostart + dstCapacity;
BYTE* op = ostart;
size_t lastCountSize;
int longOffsets = 0;
entropyWorkspace = count + (MaxSeq + 1);
entropyWkspSize -= (MaxSeq + 1) * sizeof(*count);
DEBUGLOG(5, "ZSTD_entropyCompressSeqStore_internal (nbSeq=%zu, dstCapacity=%zu)", nbSeq, dstCapacity);
ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog)));
assert(entropyWkspSize >= HUF_WORKSPACE_SIZE);
/* Compress literals */
{ size_t const numSequences = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
/* Base suspicion of uncompressibility on ratio of literals to sequences */
int const suspectUncompressible = (numSequences == 0) || (litSize / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO);
size_t const cSize = ZSTD_compressLiterals(
op, dstCapacity,
literals, litSize,
entropyWorkspace, entropyWkspSize,
&prevEntropy->huf, &nextEntropy->huf,
cctxParams->cParams.strategy,
ZSTD_literalsCompressionIsDisabled(cctxParams),
suspectUncompressible, bmi2);
FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed");
assert(cSize <= dstCapacity);
op += cSize;
}
/* Sequences Header */
RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/,
dstSize_tooSmall, "Can't fit seq hdr in output buf!");
if (nbSeq < 128) {
*op++ = (BYTE)nbSeq;
} else if (nbSeq < LONGNBSEQ) {
op[0] = (BYTE)((nbSeq>>8) + 0x80);
op[1] = (BYTE)nbSeq;
op+=2;
} else {
op[0]=0xFF;
MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ));
op+=3;
}
assert(op <= oend);
if (nbSeq==0) {
/* Copy the old tables over as if we repeated them */
ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse));
return (size_t)(op - ostart);
}
{ BYTE* const seqHead = op++;
/* build stats for sequences */
const ZSTD_symbolEncodingTypeStats_t stats =
ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq,
&prevEntropy->fse, &nextEntropy->fse,
op, oend,
strategy, count,
entropyWorkspace, entropyWkspSize);
FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!");
*seqHead = (BYTE)((stats.LLtype<<6) + (stats.Offtype<<4) + (stats.MLtype<<2));
lastCountSize = stats.lastCountSize;
op += stats.size;
longOffsets = stats.longOffsets;
}
{ size_t const bitstreamSize = ZSTD_encodeSequences(
op, (size_t)(oend - op),
CTable_MatchLength, mlCodeTable,
CTable_OffsetBits, ofCodeTable,
CTable_LitLength, llCodeTable,
sequences, nbSeq,
longOffsets, bmi2);
FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed");
op += bitstreamSize;
assert(op <= oend);
/* zstd versions <= 1.3.4 mistakenly report corruption when
* FSE_readNCount() receives a buffer < 4 bytes.
* Fixed by https://github.com/facebook/zstd/pull/1146.
* This can happen when the last set_compressed table present is 2
* bytes and the bitstream is only one byte.
* In this exceedingly rare case, we will simply emit an uncompressed
* block, since it isn't worth optimizing.
*/
if (lastCountSize && (lastCountSize + bitstreamSize) < 4) {
/* lastCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */
assert(lastCountSize + bitstreamSize == 3);
DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by "
"emitting an uncompressed block.");
return 0;
}
}
DEBUGLOG(5, "compressed block size : %u", (unsigned)(op - ostart));
return (size_t)(op - ostart);
}
static size_t
ZSTD_entropyCompressSeqStore_wExtLitBuffer(
void* dst, size_t dstCapacity,
const void* literals, size_t litSize,
size_t blockSize,
const SeqStore_t* seqStorePtr,
const ZSTD_entropyCTables_t* prevEntropy,
ZSTD_entropyCTables_t* nextEntropy,
const ZSTD_CCtx_params* cctxParams,
void* entropyWorkspace, size_t entropyWkspSize,
int bmi2)
{
size_t const cSize = ZSTD_entropyCompressSeqStore_internal(
dst, dstCapacity,
literals, litSize,
seqStorePtr, prevEntropy, nextEntropy, cctxParams,
entropyWorkspace, entropyWkspSize, bmi2);
if (cSize == 0) return 0;
/* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block.
* Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block.
*/
if ((cSize == ERROR(dstSize_tooSmall)) & (blockSize <= dstCapacity)) {
DEBUGLOG(4, "not enough dstCapacity (%zu) for ZSTD_entropyCompressSeqStore_internal()=> do not compress block", dstCapacity);
return 0; /* block not compressed */
}
FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSeqStore_internal failed");
/* Check compressibility */
{ size_t const maxCSize = blockSize - ZSTD_minGain(blockSize, cctxParams->cParams.strategy);
if (cSize >= maxCSize) return 0; /* block not compressed */
}
DEBUGLOG(5, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize);
/* libzstd decoder before > v1.5.4 is not compatible with compressed blocks of size ZSTD_BLOCKSIZE_MAX exactly.
* This restriction is indirectly already fulfilled by respecting ZSTD_minGain() condition above.
*/
assert(cSize < ZSTD_BLOCKSIZE_MAX);
return cSize;
return ZSTD_rust_entropyCompressSeqStore_internal(
dst, dstCapacity,
literals, litSize,
seqStorePtr, prevEntropy, nextEntropy,
(int)cctxParams->cParams.strategy,
ZSTD_literalsCompressionIsDisabled(cctxParams),
entropyWorkspace, entropyWkspSize, bmi2);
}
static size_t
@@ -2923,13 +2682,12 @@ ZSTD_entropyCompressSeqStore(
void* entropyWorkspace, size_t entropyWkspSize,
int bmi2)
{
return ZSTD_entropyCompressSeqStore_wExtLitBuffer(
return ZSTD_rust_entropyCompressSeqStore(
seqStorePtr, prevEntropy, nextEntropy,
(int)cctxParams->cParams.strategy,
ZSTD_literalsCompressionIsDisabled(cctxParams),
dst, dstCapacity,
seqStorePtr->litStart, (size_t)(seqStorePtr->lit - seqStorePtr->litStart),
srcSize,
seqStorePtr,
prevEntropy, nextEntropy,
cctxParams,
entropyWorkspace, entropyWkspSize,
bmi2);
}
@@ -3299,87 +3057,8 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
static size_t ZSTD_copyBlockSequences(SeqCollector* seqCollector, const SeqStore_t* seqStore, const U32 prevRepcodes[ZSTD_REP_NUM])
{
const SeqDef* inSeqs = seqStore->sequencesStart;
const size_t nbInSequences = (size_t)(seqStore->sequences - inSeqs);
const size_t nbInLiterals = (size_t)(seqStore->lit - seqStore->litStart);
ZSTD_Sequence* outSeqs = seqCollector->seqIndex == 0 ? seqCollector->seqStart : seqCollector->seqStart + seqCollector->seqIndex;
const size_t nbOutSequences = nbInSequences + 1;
size_t nbOutLiterals = 0;
Repcodes_t repcodes;
size_t i;
/* Bounds check that we have enough space for every input sequence
* and the block delimiter
*/
assert(seqCollector->seqIndex <= seqCollector->maxSequences);
RETURN_ERROR_IF(
nbOutSequences > (size_t)(seqCollector->maxSequences - seqCollector->seqIndex),
dstSize_tooSmall,
"Not enough space to copy sequences");
ZSTD_memcpy(&repcodes, prevRepcodes, sizeof(repcodes));
for (i = 0; i < nbInSequences; ++i) {
U32 rawOffset;
outSeqs[i].litLength = inSeqs[i].litLength;
outSeqs[i].matchLength = inSeqs[i].mlBase + MINMATCH;
outSeqs[i].rep = 0;
/* Handle the possible single length >= 64K
* There can only be one because we add MINMATCH to every match length,
* and blocks are at most 128K.
*/
if (i == seqStore->longLengthPos) {
if (seqStore->longLengthType == ZSTD_llt_literalLength) {
outSeqs[i].litLength += 0x10000;
} else if (seqStore->longLengthType == ZSTD_llt_matchLength) {
outSeqs[i].matchLength += 0x10000;
}
}
/* Determine the raw offset given the offBase, which may be a repcode. */
if (OFFBASE_IS_REPCODE(inSeqs[i].offBase)) {
const U32 repcode = OFFBASE_TO_REPCODE(inSeqs[i].offBase);
assert(repcode > 0);
outSeqs[i].rep = repcode;
if (outSeqs[i].litLength != 0) {
rawOffset = repcodes.rep[repcode - 1];
} else {
if (repcode == 3) {
assert(repcodes.rep[0] > 1);
rawOffset = repcodes.rep[0] - 1;
} else {
rawOffset = repcodes.rep[repcode];
}
}
} else {
rawOffset = OFFBASE_TO_OFFSET(inSeqs[i].offBase);
}
outSeqs[i].offset = rawOffset;
/* Update repcode history for the sequence */
ZSTD_updateRep(repcodes.rep,
inSeqs[i].offBase,
inSeqs[i].litLength == 0);
nbOutLiterals += outSeqs[i].litLength;
}
/* Insert last literals (if any exist) in the block as a sequence with ml == off == 0.
* If there are no last literals, then we'll emit (of: 0, ml: 0, ll: 0), which is a marker
* for the block boundary, according to the API.
*/
assert(nbInLiterals >= nbOutLiterals);
{
const size_t lastLLSize = nbInLiterals - nbOutLiterals;
outSeqs[nbInSequences].litLength = (U32)lastLLSize;
outSeqs[nbInSequences].matchLength = 0;
outSeqs[nbInSequences].offset = 0;
assert(nbOutSequences == nbInSequences + 1);
}
seqCollector->seqIndex += nbOutSequences;
assert(seqCollector->seqIndex <= seqCollector->maxSequences);
return 0;
/* The implementation lives in rust/src/zstd_compress_stats.rs. */
return ZSTD_rust_copyBlockSequences(seqCollector, seqStore, prevRepcodes);
}
/* ZSTD_sequenceBound() lives in rust/src/zstd_compress_api.rs. */
@@ -3475,178 +3154,14 @@ writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock)
DEBUGLOG(5, "writeBlockHeader: cSize: %zu blockSize: %zu lastBlock: %u", cSize, blockSize, lastBlock);
}
/** ZSTD_buildBlockEntropyStats_literals() :
* Builds entropy for the literals.
* Stores literals block type (raw, rle, compressed, repeat) and
* huffman description table to hufMetadata.
* Requires ENTROPY_WORKSPACE_SIZE workspace
* @return : size of huffman description table, or an error code
*/
static size_t
ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize,
const ZSTD_hufCTables_t* prevHuf,
ZSTD_hufCTables_t* nextHuf,
ZSTD_hufCTablesMetadata_t* hufMetadata,
const int literalsCompressionIsDisabled,
void* workspace, size_t wkspSize,
int hufFlags)
{
BYTE* const wkspStart = (BYTE*)workspace;
BYTE* const wkspEnd = wkspStart + wkspSize;
BYTE* const countWkspStart = wkspStart;
unsigned* const countWksp = (unsigned*)workspace;
const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned);
BYTE* const nodeWksp = countWkspStart + countWkspSize;
const size_t nodeWkspSize = (size_t)(wkspEnd - nodeWksp);
unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX;
unsigned huffLog = LitHufLog;
HUF_repeat repeat = prevHuf->repeatMode;
DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_literals (srcSize=%zu)", srcSize);
/* Prepare nextEntropy assuming reusing the existing table */
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
if (literalsCompressionIsDisabled) {
DEBUGLOG(5, "set_basic - disabled");
hufMetadata->hType = set_basic;
return 0;
}
/* small ? don't even attempt compression (speed opt) */
#ifndef COMPRESS_LITERALS_SIZE_MIN
# define COMPRESS_LITERALS_SIZE_MIN 63 /* heuristic */
#endif
{ size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN;
if (srcSize <= minLitSize) {
DEBUGLOG(5, "set_basic - too small");
hufMetadata->hType = set_basic;
return 0;
} }
/* Scan input and build symbol stats */
{ size_t const largest =
HIST_count_wksp (countWksp, &maxSymbolValue,
(const BYTE*)src, srcSize,
workspace, wkspSize);
FORWARD_IF_ERROR(largest, "HIST_count_wksp failed");
if (largest == srcSize) {
/* only one literal symbol */
DEBUGLOG(5, "set_rle");
hufMetadata->hType = set_rle;
return 0;
}
if (largest <= (srcSize >> 7)+4) {
/* heuristic: likely not compressible */
DEBUGLOG(5, "set_basic - no gain");
hufMetadata->hType = set_basic;
return 0;
} }
/* Validate the previous Huffman table */
if (repeat == HUF_repeat_check
&& !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) {
repeat = HUF_repeat_none;
}
/* Build Huffman Tree */
ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable));
huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, nodeWksp, nodeWkspSize, nextHuf->CTable, countWksp, hufFlags);
assert(huffLog <= LitHufLog);
{ size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp,
maxSymbolValue, huffLog,
nodeWksp, nodeWkspSize);
FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp");
huffLog = (U32)maxBits;
}
{ /* Build and write the CTable */
size_t const newCSize = HUF_estimateCompressedSize(
(HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue);
size_t const hSize = HUF_writeCTable_wksp(
hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer),
(HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog,
nodeWksp, nodeWkspSize);
/* Check against repeating the previous CTable */
if (repeat != HUF_repeat_none) {
size_t const oldCSize = HUF_estimateCompressedSize(
(HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue);
if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) {
DEBUGLOG(5, "set_repeat - smaller");
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
hufMetadata->hType = set_repeat;
return 0;
} }
if (newCSize + hSize >= srcSize) {
DEBUGLOG(5, "set_basic - no gains");
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
hufMetadata->hType = set_basic;
return 0;
}
DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize);
hufMetadata->hType = set_compressed;
nextHuf->repeatMode = HUF_repeat_check;
return hSize;
}
}
/* ZSTD_buildDummySequencesStatistics():
* Returns a ZSTD_symbolEncodingTypeStats_t with all encoding types as set_basic,
* and updates nextEntropy to the appropriate repeatMode.
*/
static ZSTD_symbolEncodingTypeStats_t
ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy)
{
ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0, 0};
nextEntropy->litlength_repeatMode = FSE_repeat_none;
nextEntropy->offcode_repeatMode = FSE_repeat_none;
nextEntropy->matchlength_repeatMode = FSE_repeat_none;
return stats;
}
/** ZSTD_buildBlockEntropyStats_sequences() :
* Builds entropy for the sequences.
* Stores symbol compression modes and fse table to fseMetadata.
* Requires ENTROPY_WORKSPACE_SIZE wksp.
* @return : size of fse tables or error code */
static size_t
ZSTD_buildBlockEntropyStats_sequences(
const SeqStore_t* seqStorePtr,
const ZSTD_fseCTables_t* prevEntropy,
ZSTD_fseCTables_t* nextEntropy,
const ZSTD_CCtx_params* cctxParams,
ZSTD_fseCTablesMetadata_t* fseMetadata,
void* workspace, size_t wkspSize)
{
ZSTD_strategy const strategy = cctxParams->cParams.strategy;
size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
BYTE* const ostart = fseMetadata->fseTablesBuffer;
BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer);
BYTE* op = ostart;
unsigned* countWorkspace = (unsigned*)workspace;
unsigned* entropyWorkspace = countWorkspace + (MaxSeq + 1);
size_t entropyWorkspaceSize = wkspSize - (MaxSeq + 1) * sizeof(*countWorkspace);
ZSTD_symbolEncodingTypeStats_t stats;
DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_sequences (nbSeq=%zu)", nbSeq);
stats = nbSeq != 0 ? ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq,
prevEntropy, nextEntropy, op, oend,
strategy, countWorkspace,
entropyWorkspace, entropyWorkspaceSize)
: ZSTD_buildDummySequencesStatistics(nextEntropy);
FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!");
fseMetadata->llType = (SymbolEncodingType_e) stats.LLtype;
fseMetadata->ofType = (SymbolEncodingType_e) stats.Offtype;
fseMetadata->mlType = (SymbolEncodingType_e) stats.MLtype;
fseMetadata->lastCountSize = stats.lastCountSize;
return stats.size;
}
/** ZSTD_buildBlockEntropyStats() :
* Builds entropy for the block.
* Requires workspace size ENTROPY_WORKSPACE_SIZE
* @return : 0 on success, or an error code
* Note : also employed in superblock
*
* The implementation, together with its literals/sequences/dummy helpers,
* lives in rust/src/zstd_compress_stats.rs.
*/
size_t ZSTD_buildBlockEntropyStats(
const SeqStore_t* seqStorePtr,
@@ -3656,26 +3171,12 @@ size_t ZSTD_buildBlockEntropyStats(
ZSTD_entropyCTablesMetadata_t* entropyMetadata,
void* workspace, size_t wkspSize)
{
size_t const litSize = (size_t)(seqStorePtr->lit - seqStorePtr->litStart);
int const huf_useOptDepth = (cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD);
int const hufFlags = huf_useOptDepth ? HUF_flags_optimalDepth : 0;
entropyMetadata->hufMetadata.hufDesSize =
ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize,
&prevEntropy->huf, &nextEntropy->huf,
&entropyMetadata->hufMetadata,
ZSTD_literalsCompressionIsDisabled(cctxParams),
workspace, wkspSize, hufFlags);
FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed");
entropyMetadata->fseMetadata.fseTablesSize =
ZSTD_buildBlockEntropyStats_sequences(seqStorePtr,
&prevEntropy->fse, &nextEntropy->fse,
cctxParams,
&entropyMetadata->fseMetadata,
workspace, wkspSize);
FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildBlockEntropyStats_sequences failed");
return 0;
return ZSTD_rust_buildBlockEntropyStats(
seqStorePtr, prevEntropy, nextEntropy,
(int)cctxParams->cParams.strategy,
ZSTD_literalsCompressionIsDisabled(cctxParams),
entropyMetadata,
workspace, wkspSize);
}
/* Returns the size estimate for the literals section (header + content) of a block */
lib/compress/zstd_compress_superblock.c
+3 -2
View File
@@ -29,7 +29,7 @@ size_t ZSTD_rust_compressSuperBlock(
const SeqStore_t* seqStore,
const ZSTD_compressedBlockState_t* prevCBlock,
ZSTD_compressedBlockState_t* nextCBlock,
const ZSTD_CCtx_params* cctxParams,
int strategy, int disableLiteralCompression,
void* workspace, size_t wkspSize,
int bmi2, U32 windowLog, size_t targetCBlockSize,
void* dst, size_t dstCapacity,
@@ -45,7 +45,8 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
&zc->seqStore,
zc->blockState.prevCBlock,
zc->blockState.nextCBlock,
&zc->appliedParams,
(int)zc->appliedParams.cParams.strategy,
ZSTD_literalsCompressionIsDisabled(&zc->appliedParams),
zc->tmpWorkspace, zc->tmpWkspSize,
zc->bmi2, zc->appliedParams.cParams.windowLog,
zc->appliedParams.targetCBlockSize,
rust/README.md
+7
View File
@@ -31,6 +31,13 @@ zstd ABI:
- `zstd_presplit` chooses split points for full compression blocks.
- `zstd_compress_literals` emits raw, RLE, and Huffman literal sections
while preserving the compressor's Huffman-table repeat state.
- `zstd_compress_stats` converts stored sequences into symbol codes,
selects each block's symbol encoding types, compresses a seqStore's
literals and sequences into a compressed-block body, builds the block
entropy statistics shared with the superblock writer and the block
splitter, and exports collected sequences in the public `ZSTD_Sequence`
format. Its C shims extract the sequence store, the entropy-table
leaves, and the two `ZSTD_CCtx_params` scalars these paths read.
- `zstd_compress_frame` serializes frame headers, skippable frames, and the
last empty block; it takes scalar frame parameters so the C-owned
`ZSTD_CCtx_params` layout never crosses the language boundary.
rust/src/lib.rs
+2
View File
@@ -32,6 +32,8 @@ pub mod zstd_compress_params;
#[cfg(feature = "compression")]
pub mod zstd_compress_sequences;
#[cfg(feature = "compression")]
pub mod zstd_compress_stats;
#[cfg(feature = "compression")]
pub mod zstd_compress_superblock;
#[cfg(feature = "decompression")]
pub mod zstd_ddict;
rust/src/zstd_compress_literals.rs
+4 -2
View File
@@ -65,7 +65,7 @@ unsafe fn copy_bytes(dst: *mut u8, src: *const u8, size: usize) {
}
#[inline]
unsafe fn copy_huf_tables(dst: *mut ZSTD_hufCTables_t, src: *const ZSTD_hufCTables_t) {
pub(crate) unsafe fn copy_huf_tables(dst: *mut ZSTD_hufCTables_t, src: *const ZSTD_hufCTables_t) {
// Copy as bytes, not as a Rust struct: C's memcpy preserves the four tail
// padding bytes that this layout has on 64-bit targets. Some C callers
// copy or compare the complete entropy state, so leaving those bytes from
@@ -192,8 +192,10 @@ fn min_literals_to_compress(strategy: c_int, huf_repeat: c_int) -> usize {
}
}
/// C's `ZSTD_minGain()`: minimum compression gain required to emit a
/// compressed block or compressed literals section.
#[inline]
fn min_gain(src_size: usize, strategy: c_int) -> usize {
pub(crate) fn min_gain(src_size: usize, strategy: c_int) -> usize {
let min_log = if strategy >= ZSTD_BTULTRA {
strategy.saturating_sub(1) as u32
} else {
rust/src/zstd_compress_stats.rs
+1549
View File
@@ -0,0 +1,1549 @@
#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
//! Sequence statistics and seqStore entropy compression.
//!
//! This module translates the block entropy-statistics layer of
//! `zstd_compress.c`: `ZSTD_seqToCodes()`, `ZSTD_buildSequencesStatistics()`,
//! `ZSTD_entropyCompressSeqStore*()`, `ZSTD_buildBlockEntropyStats*()`, and
//! `ZSTD_copyBlockSequences()`. The `ZSTD_CCtx` and `ZSTD_CCtx_params`
//! layouts stay private to C: the C shims extract the sequence store, the
//! entropy-table leaves, and the two parameter scalars these paths read
//! (the compression strategy and the literals-compression switch). Block
//! dispatch, block splitting, and the block-size estimation heuristics also
//! remain in C for a later slice.
use crate::bits::ZSTD_highbit32;
use crate::common::{
DEFAULT_MAX_OFF, LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, LL_FSE_LOG, LONGNBSEQ, MAX_LL, MAX_ML,
MAX_OFF, MAX_SEQ, MINMATCH, ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, ML_FSE_LOG, OFF_FSE_LOG,
OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, ZSTD_MAX_FSE_HEADERS_SIZE, ZSTD_MAX_HUF_HEADER_SIZE,
ZSTD_REP_NUM,
};
use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
use crate::hist::{HIST_countFast_wksp, HIST_count_wksp};
use crate::huf_compress::{
HUF_buildCTable_wksp, HUF_estimateCompressedSize, HUF_optimalTableLog, HUF_validateCTable,
HUF_writeCTable_wksp,
};
use crate::mem::{MEM_32bits, MEM_writeLE16};
use crate::zstd_compress_literals::{
copy_huf_tables, min_gain, ZSTD_compressLiterals, ZSTD_hufCTables_t,
};
use crate::zstd_compress_sequences::{
SeqDef, ZSTD_buildCTable, ZSTD_encodeSequences, ZSTD_selectEncodingType,
};
use std::ffi::c_void;
use std::mem::{size_of, size_of_val};
use std::os::raw::{c_int, c_uint};
use std::ptr;
const SET_BASIC: c_int = 0;
const SET_RLE: c_int = 1;
const SET_COMPRESSED: c_int = 2;
const SET_REPEAT: c_int = 3;
const FSE_REPEAT_NONE: c_int = 0;
const HUF_REPEAT_NONE: c_int = 0;
const HUF_REPEAT_CHECK: c_int = 1;
const HUF_REPEAT_VALID: c_int = 2;
const HUF_FLAGS_OPTIMAL_DEPTH: c_int = 1 << 1;
const HUF_OPTIMAL_DEPTH_THRESHOLD: c_int = 8; /* ZSTD_btultra */
const ZSTD_DEFAULT_DISALLOWED: c_int = 0;
const ZSTD_DEFAULT_ALLOWED: c_int = 1;
const HUF_SYMBOLVALUE_MAX: c_uint = 255;
const LIT_HUF_LOG: c_uint = 11;
const COMPRESS_LITERALS_SIZE_MIN: usize = 63;
const SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO: usize = 20;
const STREAM_ACCUMULATOR_MIN_32: u32 = 25;
const STREAM_ACCUMULATOR_MIN_64: u32 = 57;
const LL_DELTA_CODE: u32 = 19;
const ML_DELTA_CODE: u32 = 36;
pub(crate) const ZSTD_LLT_LITERAL_LENGTH: c_int = 1;
pub(crate) const ZSTD_LLT_MATCH_LENGTH: c_int = 2;
/// `FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)` from the C headers.
pub const OFF_CTABLE_SIZE: usize = 1 + (1 << (OFF_FSE_LOG - 1)) + ((MAX_OFF + 1) * 2);
/// `FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)` from the C headers.
pub const ML_CTABLE_SIZE: usize = 1 + (1 << (ML_FSE_LOG - 1)) + ((MAX_ML + 1) * 2);
/// `FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)` from the C headers.
pub const LL_CTABLE_SIZE: usize = 1 + (1 << (LL_FSE_LOG - 1)) + ((MAX_LL + 1) * 2);
/// ABI-compatible `SeqStore_t` leaf layout from `zstd_compress_internal.h`.
///
/// The C context itself remains opaque; its shims pass the `seqStore` member
/// directly, so this small, stable hot-path structure is the only sequence
/// storage representation crossing into Rust.
#[repr(C)]
pub struct SeqStore_t {
pub sequencesStart: *mut SeqDef,
pub sequences: *mut SeqDef,
pub litStart: *mut u8,
pub lit: *mut u8,
pub llCode: *mut u8,
pub mlCode: *mut u8,
pub ofCode: *mut u8,
pub maxNbSeq: usize,
pub maxNbLit: usize,
pub longLengthType: c_int,
pub longLengthPos: u32,
}
/// C's `ZSTD_fseCTables_t`. The table element type is `FSE_CTable`, an
/// `unsigned`, and the table lengths are the header macros expanded above.
#[repr(C)]
pub struct ZSTD_fseCTables_t {
pub offcodeCTable: [u32; OFF_CTABLE_SIZE],
pub matchlengthCTable: [u32; ML_CTABLE_SIZE],
pub litlengthCTable: [u32; LL_CTABLE_SIZE],
pub offcode_repeatMode: c_int,
pub matchlength_repeatMode: c_int,
pub litlength_repeatMode: c_int,
}
/// C's `ZSTD_entropyCTables_t`.
#[repr(C)]
pub struct ZSTD_entropyCTables_t {
pub huf: ZSTD_hufCTables_t,
pub fse: ZSTD_fseCTables_t,
}
/// C's `ZSTD_compressedBlockState_t`.
#[repr(C)]
pub struct ZSTD_compressedBlockState_t {
pub entropy: ZSTD_entropyCTables_t,
pub rep: [u32; ZSTD_REP_NUM],
}
/// C's `ZSTD_hufCTablesMetadata_t`.
#[repr(C)]
pub struct ZSTD_hufCTablesMetadata_t {
pub hType: c_int,
pub hufDesBuffer: [u8; ZSTD_MAX_HUF_HEADER_SIZE],
pub hufDesSize: usize,
}
/// C's `ZSTD_fseCTablesMetadata_t`.
#[repr(C)]
pub struct ZSTD_fseCTablesMetadata_t {
pub llType: c_int,
pub ofType: c_int,
pub mlType: c_int,
pub fseTablesBuffer: [u8; ZSTD_MAX_FSE_HEADERS_SIZE],
pub fseTablesSize: usize,
pub lastCountSize: usize,
}
/// C's `ZSTD_entropyCTablesMetadata_t`.
#[repr(C)]
pub struct ZSTD_entropyCTablesMetadata_t {
pub hufMetadata: ZSTD_hufCTablesMetadata_t,
pub fseMetadata: ZSTD_fseCTablesMetadata_t,
}
/// Public `ZSTD_Sequence` from `zstd.h`.
#[repr(C)]
#[derive(Clone, Copy)]
pub struct ZSTD_Sequence {
pub offset: u32,
pub litLength: u32,
pub matchLength: u32,
pub rep: u32,
}
/// C's `SeqCollector` leaf from `zstd_compress_internal.h`.
#[repr(C)]
pub struct SeqCollector {
pub collectSequences: c_int,
pub seqStart: *mut ZSTD_Sequence,
pub seqIndex: usize,
pub maxSequences: usize,
}
/// Result of `build_sequences_statistics()`; mirrors C's
/// `ZSTD_symbolEncodingTypeStats_t`, which no longer crosses the boundary.
pub(crate) struct SymbolEncodingTypeStats {
pub ll_type: u32,
pub off_type: u32,
pub ml_type: u32,
pub size: usize,
/// Accounts for a bug in zstd 1.3.4; see
/// `entropy_compress_seq_store_internal()`.
pub last_count_size: usize,
pub long_offsets: c_int,
}
/// C's `ZSTD_updateRep()` repeat-offset history update.
#[inline]
pub(crate) fn update_rep(reps: &mut [u32; 3], off_base: u32, literal_length_is_zero: bool) {
if off_base > 3 {
reps[2] = reps[1];
reps[1] = reps[0];
reps[0] = off_base - 3;
return;
}
let rep_code = off_base - 1 + u32::from(literal_length_is_zero);
if rep_code == 0 {
return;
}
let current_offset = if rep_code == 3 {
reps[0].wrapping_sub(1)
} else {
reps[rep_code as usize]
};
reps[2] = if rep_code >= 2 { reps[1] } else { reps[2] };
reps[1] = reps[0];
reps[0] = current_offset;
}
#[inline]
fn ll_code(lit_length: u32) -> u32 {
const LL_CODE: [u8; 64] = [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20,
20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 23, 23, 23,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
];
if lit_length > 63 {
ZSTD_highbit32(lit_length) + LL_DELTA_CODE
} else {
LL_CODE[lit_length as usize] as u32
}
}
#[inline]
fn ml_code(ml_base: u32) -> u32 {
const ML_CODE: [u8; 128] = [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
25, 26, 27, 28, 29, 30, 31, 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40,
40, 40, 40, 40, 40, 40, 40, 40, 40, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
41, 41, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
];
if ml_base > 127 {
ZSTD_highbit32(ml_base) + ML_DELTA_CODE
} else {
ML_CODE[ml_base as usize] as u32
}
}
#[inline]
unsafe fn copy_fse_tables(dst: *mut ZSTD_fseCTables_t, src: *const ZSTD_fseCTables_t) {
unsafe {
ptr::copy_nonoverlapping(
src.cast::<u8>(),
dst.cast::<u8>(),
size_of::<ZSTD_fseCTables_t>(),
)
};
}
/// Converts the stored lengths and distances into symbol codes.
///
/// Retains its original exported name: the C dictionary builder and the
/// decodecorpus test tool link against it directly.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_seqToCodes(seq_store: *const SeqStore_t) -> c_int {
let store = unsafe { &*seq_store };
let sequences = store.sequencesStart;
let nb_seq = unsafe { store.sequences.offset_from(sequences) } as usize;
let mut long_offsets = 0;
debug_assert!(nb_seq <= store.maxNbSeq);
for index in 0..nb_seq {
let sequence = unsafe { sequences.add(index).read() };
let of_code = ZSTD_highbit32(sequence.offBase);
unsafe {
*store.llCode.add(index) = ll_code(sequence.litLength as u32) as u8;
*store.ofCode.add(index) = of_code as u8;
*store.mlCode.add(index) = ml_code(sequence.mlBase as u32) as u8;
}
debug_assert!(MEM_32bits() || of_code < STREAM_ACCUMULATOR_MIN_64);
if MEM_32bits() && of_code >= STREAM_ACCUMULATOR_MIN_32 {
long_offsets = 1;
}
}
if store.longLengthType == ZSTD_LLT_LITERAL_LENGTH {
unsafe { *store.llCode.add(store.longLengthPos as usize) = MAX_LL as u8 };
}
if store.longLengthType == ZSTD_LLT_MATCH_LENGTH {
unsafe { *store.mlCode.add(store.longLengthPos as usize) = MAX_ML as u8 };
}
long_offsets
}
/// C's `ZSTD_buildSequencesStatistics()`.
///
/// Returns the statistics with a zstd error code in the `size` field on
/// failure, and updates `next_entropy` as a side effect. `nb_seq` must be
/// greater than zero, and the entropy workspace must hold at least
/// `ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1) * sizeof(U32)` bytes.
#[allow(clippy::too_many_arguments)]
pub(crate) unsafe fn build_sequences_statistics(
seq_store: *const SeqStore_t,
nb_seq: usize,
prev_entropy: *const ZSTD_fseCTables_t,
next_entropy: *mut ZSTD_fseCTables_t,
dst: *mut u8,
dst_capacity: usize,
strategy: c_int,
count_workspace: *mut c_uint,
entropy_workspace: *mut c_void,
entropy_wksp_size: usize,
) -> SymbolEncodingTypeStats {
let store = unsafe { &*seq_store };
let mut stats = SymbolEncodingTypeStats {
ll_type: 0,
off_type: 0,
ml_type: 0,
size: 0,
last_count_size: 0,
long_offsets: 0,
};
let mut written = 0usize;
/* convert length/distances into codes */
stats.long_offsets = unsafe { ZSTD_seqToCodes(seq_store) };
debug_assert!(nb_seq != 0); /* ZSTD_selectEncodingType() divides by nb_seq */
/* build CTable for Literal Lengths */
{
let mut max = MAX_LL as c_uint;
let most_frequent = unsafe {
HIST_countFast_wksp(
count_workspace,
&mut max,
store.llCode.cast::<c_void>(),
nb_seq,
entropy_workspace,
entropy_wksp_size,
)
}; /* can't fail */
unsafe { (*next_entropy).litlength_repeatMode = (*prev_entropy).litlength_repeatMode };
stats.ll_type = unsafe {
ZSTD_selectEncodingType(
ptr::addr_of_mut!((*next_entropy).litlength_repeatMode),
count_workspace,
max,
most_frequent,
nb_seq,
LL_FSE_LOG as c_uint,
ptr::addr_of!((*prev_entropy).litlengthCTable).cast::<u32>(),
LL_DEFAULT_NORM.as_ptr(),
LL_DEFAULT_NORM_LOG,
ZSTD_DEFAULT_ALLOWED,
strategy,
)
} as u32;
let count_size = unsafe {
ZSTD_buildCTable(
dst.add(written).cast::<c_void>(),
dst_capacity - written,
ptr::addr_of_mut!((*next_entropy).litlengthCTable).cast::<u32>(),
LL_FSE_LOG as u32,
stats.ll_type as c_int,
count_workspace,
max,
store.llCode,
nb_seq,
LL_DEFAULT_NORM.as_ptr(),
LL_DEFAULT_NORM_LOG,
MAX_LL as u32,
ptr::addr_of!((*prev_entropy).litlengthCTable).cast::<u32>(),
size_of::<[u32; LL_CTABLE_SIZE]>(),
entropy_workspace,
entropy_wksp_size,
)
};
if ERR_isError(count_size) {
stats.size = count_size;
return stats;
}
if stats.ll_type == SET_COMPRESSED as u32 {
stats.last_count_size = count_size;
}
written += count_size;
}
/* build CTable for Offsets */
{
let mut max = MAX_OFF as c_uint;
let most_frequent = unsafe {
HIST_countFast_wksp(
count_workspace,
&mut max,
store.ofCode.cast::<c_void>(),
nb_seq,
entropy_workspace,
entropy_wksp_size,
)
}; /* can't fail */
/* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */
let default_policy = if max <= DEFAULT_MAX_OFF as c_uint {
ZSTD_DEFAULT_ALLOWED
} else {
ZSTD_DEFAULT_DISALLOWED
};
unsafe { (*next_entropy).offcode_repeatMode = (*prev_entropy).offcode_repeatMode };
stats.off_type = unsafe {
ZSTD_selectEncodingType(
ptr::addr_of_mut!((*next_entropy).offcode_repeatMode),
count_workspace,
max,
most_frequent,
nb_seq,
OFF_FSE_LOG as c_uint,
ptr::addr_of!((*prev_entropy).offcodeCTable).cast::<u32>(),
OF_DEFAULT_NORM.as_ptr(),
OF_DEFAULT_NORM_LOG,
default_policy,
strategy,
)
} as u32;
let count_size = unsafe {
ZSTD_buildCTable(
dst.add(written).cast::<c_void>(),
dst_capacity - written,
ptr::addr_of_mut!((*next_entropy).offcodeCTable).cast::<u32>(),
OFF_FSE_LOG as u32,
stats.off_type as c_int,
count_workspace,
max,
store.ofCode,
nb_seq,
OF_DEFAULT_NORM.as_ptr(),
OF_DEFAULT_NORM_LOG,
DEFAULT_MAX_OFF as u32,
ptr::addr_of!((*prev_entropy).offcodeCTable).cast::<u32>(),
size_of::<[u32; OFF_CTABLE_SIZE]>(),
entropy_workspace,
entropy_wksp_size,
)
};
if ERR_isError(count_size) {
stats.size = count_size;
return stats;
}
if stats.off_type == SET_COMPRESSED as u32 {
stats.last_count_size = count_size;
}
written += count_size;
}
/* build CTable for MatchLengths */
{
let mut max = MAX_ML as c_uint;
let most_frequent = unsafe {
HIST_countFast_wksp(
count_workspace,
&mut max,
store.mlCode.cast::<c_void>(),
nb_seq,
entropy_workspace,
entropy_wksp_size,
)
}; /* can't fail */
unsafe { (*next_entropy).matchlength_repeatMode = (*prev_entropy).matchlength_repeatMode };
stats.ml_type = unsafe {
ZSTD_selectEncodingType(
ptr::addr_of_mut!((*next_entropy).matchlength_repeatMode),
count_workspace,
max,
most_frequent,
nb_seq,
ML_FSE_LOG as c_uint,
ptr::addr_of!((*prev_entropy).matchlengthCTable).cast::<u32>(),
ML_DEFAULT_NORM.as_ptr(),
ML_DEFAULT_NORM_LOG,
ZSTD_DEFAULT_ALLOWED,
strategy,
)
} as u32;
let count_size = unsafe {
ZSTD_buildCTable(
dst.add(written).cast::<c_void>(),
dst_capacity - written,
ptr::addr_of_mut!((*next_entropy).matchlengthCTable).cast::<u32>(),
ML_FSE_LOG as u32,
stats.ml_type as c_int,
count_workspace,
max,
store.mlCode,
nb_seq,
ML_DEFAULT_NORM.as_ptr(),
ML_DEFAULT_NORM_LOG,
MAX_ML as u32,
ptr::addr_of!((*prev_entropy).matchlengthCTable).cast::<u32>(),
size_of::<[u32; ML_CTABLE_SIZE]>(),
entropy_workspace,
entropy_wksp_size,
)
};
if ERR_isError(count_size) {
stats.size = count_size;
return stats;
}
if stats.ml_type == SET_COMPRESSED as u32 {
stats.last_count_size = count_size;
}
written += count_size;
}
stats.size = written;
stats
}
/// C's `ZSTD_entropyCompressSeqStore_internal()`: compresses literals and
/// sequences into one compressed-block body.
#[allow(clippy::too_many_arguments)]
unsafe fn entropy_compress_seq_store_internal(
dst: *mut c_void,
dst_capacity: usize,
literals: *const c_void,
lit_size: usize,
seq_store: *const SeqStore_t,
prev_entropy: *const ZSTD_entropyCTables_t,
next_entropy: *mut ZSTD_entropyCTables_t,
strategy: c_int,
disable_literal_compression: c_int,
entropy_workspace: *mut c_void,
entropy_wksp_size: usize,
bmi2: c_int,
) -> usize {
let count = entropy_workspace.cast::<c_uint>();
let entropy_workspace = unsafe { count.add(MAX_SEQ + 1) }.cast::<c_void>();
let entropy_wksp_size = entropy_wksp_size.wrapping_sub((MAX_SEQ + 1) * size_of::<c_uint>());
let store = unsafe { &*seq_store };
let sequences = store.sequencesStart;
let nb_seq = unsafe { store.sequences.offset_from(sequences) } as usize;
let ostart = dst.cast::<u8>();
let mut written = 0usize;
let last_count_size;
let long_offsets;
/* Compress literals */
{
/* Base suspicion of uncompressibility on ratio of literals to sequences */
let suspect_uncompressible =
c_int::from(nb_seq == 0 || lit_size / nb_seq >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO);
let c_size = unsafe {
ZSTD_compressLiterals(
dst,
dst_capacity,
literals,
lit_size,
entropy_workspace,
entropy_wksp_size,
ptr::addr_of!((*prev_entropy).huf),
ptr::addr_of_mut!((*next_entropy).huf),
strategy,
disable_literal_compression,
suspect_uncompressible,
bmi2,
)
};
if ERR_isError(c_size) {
return c_size;
}
debug_assert!(c_size <= dst_capacity);
written += c_size;
}
/* Sequences Header */
if (dst_capacity as isize).wrapping_sub(written as isize) < 3 /* max nbSeq Size */ + 1
/* seqHead */
{
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
unsafe {
if nb_seq < 128 {
*ostart.add(written) = nb_seq as u8;
written += 1;
} else if nb_seq < LONGNBSEQ as usize {
*ostart.add(written) = ((nb_seq >> 8) + 0x80) as u8;
*ostart.add(written + 1) = nb_seq as u8;
written += 2;
} else {
*ostart.add(written) = 0xFF;
MEM_writeLE16(
ostart.add(written + 1).cast::<c_void>(),
(nb_seq - LONGNBSEQ as usize) as u16,
);
written += 3;
}
}
if nb_seq == 0 {
/* Copy the old tables over as if we repeated them */
unsafe {
copy_fse_tables(
ptr::addr_of_mut!((*next_entropy).fse),
ptr::addr_of!((*prev_entropy).fse),
)
};
return written;
}
{
let seq_head = written;
written += 1;
/* build stats for sequences */
let stats = unsafe {
build_sequences_statistics(
seq_store,
nb_seq,
ptr::addr_of!((*prev_entropy).fse),
ptr::addr_of_mut!((*next_entropy).fse),
ostart.add(written),
dst_capacity - written,
strategy,
count,
entropy_workspace,
entropy_wksp_size,
)
};
if ERR_isError(stats.size) {
return stats.size;
}
unsafe {
*ostart.add(seq_head) =
((stats.ll_type << 6) + (stats.off_type << 4) + (stats.ml_type << 2)) as u8;
}
last_count_size = stats.last_count_size;
written += stats.size;
long_offsets = stats.long_offsets;
}
{
let bitstream_size = unsafe {
ZSTD_encodeSequences(
ostart.add(written).cast::<c_void>(),
dst_capacity - written,
ptr::addr_of!((*next_entropy).fse.matchlengthCTable).cast::<u32>(),
store.mlCode,
ptr::addr_of!((*next_entropy).fse.offcodeCTable).cast::<u32>(),
store.ofCode,
ptr::addr_of!((*next_entropy).fse.litlengthCTable).cast::<u32>(),
store.llCode,
sequences,
nb_seq,
long_offsets,
bmi2,
)
};
if ERR_isError(bitstream_size) {
return bitstream_size;
}
written += bitstream_size;
/* zstd versions <= 1.3.4 mistakenly report corruption when
* FSE_readNCount() receives a buffer < 4 bytes.
* Fixed by https://github.com/facebook/zstd/pull/1146.
* This can happen when the last set_compressed table present is 2
* bytes and the bitstream is only one byte.
* In this exceedingly rare case, we will simply emit an uncompressed
* block, since it isn't worth optimizing.
*/
if last_count_size != 0 && last_count_size + bitstream_size < 4 {
/* last_count_size >= 2 && bitstream_size > 0 ==> last_count_size == 3 */
debug_assert!(last_count_size + bitstream_size == 3);
return 0;
}
}
written
}
/// C's `ZSTD_entropyCompressSeqStore_wExtLitBuffer()`: adds the
/// incompressibility fallbacks around the internal entropy pass.
#[allow(clippy::too_many_arguments)]
unsafe fn entropy_compress_seq_store_w_ext_lit_buffer(
dst: *mut c_void,
dst_capacity: usize,
literals: *const c_void,
lit_size: usize,
block_size: usize,
seq_store: *const SeqStore_t,
prev_entropy: *const ZSTD_entropyCTables_t,
next_entropy: *mut ZSTD_entropyCTables_t,
strategy: c_int,
disable_literal_compression: c_int,
entropy_workspace: *mut c_void,
entropy_wksp_size: usize,
bmi2: c_int,
) -> usize {
let c_size = unsafe {
entropy_compress_seq_store_internal(
dst,
dst_capacity,
literals,
lit_size,
seq_store,
prev_entropy,
next_entropy,
strategy,
disable_literal_compression,
entropy_workspace,
entropy_wksp_size,
bmi2,
)
};
if c_size == 0 {
return 0;
}
/* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block.
* Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block.
*/
if c_size == ERROR(ZstdErrorCode::DstSizeTooSmall) && block_size <= dst_capacity {
return 0; /* block not compressed */
}
if ERR_isError(c_size) {
return c_size;
}
/* Check compressibility */
let max_c_size = block_size.wrapping_sub(min_gain(block_size, strategy));
if c_size >= max_c_size {
return 0; /* block not compressed */
}
c_size
}
/// C ABI wrapper for the external-literal-buffer entropy pass; the C shim
/// keeps the `ZSTD_entropyCompressSeqStore_internal()` name and extracts the
/// two `ZSTD_CCtx_params` scalars.
#[allow(clippy::too_many_arguments)]
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_entropyCompressSeqStore_internal(
dst: *mut c_void,
dst_capacity: usize,
literals: *const c_void,
lit_size: usize,
seq_store: *const SeqStore_t,
prev_entropy: *const ZSTD_entropyCTables_t,
next_entropy: *mut ZSTD_entropyCTables_t,
strategy: c_int,
disable_literal_compression: c_int,
entropy_workspace: *mut c_void,
entropy_wksp_size: usize,
bmi2: c_int,
) -> usize {
unsafe {
entropy_compress_seq_store_internal(
dst,
dst_capacity,
literals,
lit_size,
seq_store,
prev_entropy,
next_entropy,
strategy,
disable_literal_compression,
entropy_workspace,
entropy_wksp_size,
bmi2,
)
}
}
/// C ABI implementation behind the `ZSTD_entropyCompressSeqStore()` shim.
#[allow(clippy::too_many_arguments)]
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_entropyCompressSeqStore(
seq_store: *const SeqStore_t,
prev_entropy: *const ZSTD_entropyCTables_t,
next_entropy: *mut ZSTD_entropyCTables_t,
strategy: c_int,
disable_literal_compression: c_int,
dst: *mut c_void,
dst_capacity: usize,
src_size: usize,
entropy_workspace: *mut c_void,
entropy_wksp_size: usize,
bmi2: c_int,
) -> usize {
let store = unsafe { &*seq_store };
let lit_size = unsafe { store.lit.offset_from(store.litStart) } as usize;
unsafe {
entropy_compress_seq_store_w_ext_lit_buffer(
dst,
dst_capacity,
store.litStart.cast::<c_void>(),
lit_size,
src_size,
seq_store,
prev_entropy,
next_entropy,
strategy,
disable_literal_compression,
entropy_workspace,
entropy_wksp_size,
bmi2,
)
}
}
/// C's `ZSTD_buildBlockEntropyStats_literals()`.
///
/// Stores the literals block type and Huffman description table into
/// `huf_metadata` and returns the description size, or an error code.
#[allow(clippy::too_many_arguments)]
unsafe fn build_block_entropy_stats_literals(
src: *const c_void,
src_size: usize,
prev_huf: *const ZSTD_hufCTables_t,
next_huf: *mut ZSTD_hufCTables_t,
huf_metadata: *mut ZSTD_hufCTablesMetadata_t,
literals_compression_is_disabled: c_int,
workspace: *mut c_void,
wksp_size: usize,
huf_flags: c_int,
) -> usize {
let count_wksp = workspace.cast::<c_uint>();
let count_wksp_size = (HUF_SYMBOLVALUE_MAX as usize + 1) * size_of::<c_uint>();
let node_wksp = unsafe { workspace.cast::<u8>().add(count_wksp_size) }.cast::<c_void>();
let node_wksp_size = wksp_size.wrapping_sub(count_wksp_size);
let mut max_symbol_value = HUF_SYMBOLVALUE_MAX;
let mut huff_log = LIT_HUF_LOG;
let mut repeat = unsafe { (*prev_huf).repeatMode };
/* Prepare nextEntropy assuming reusing the existing table */
unsafe { copy_huf_tables(next_huf, prev_huf) };
if literals_compression_is_disabled != 0 {
unsafe { (*huf_metadata).hType = SET_BASIC };
return 0;
}
/* small ? don't even attempt compression (speed opt) */
{
let min_lit_size = if repeat == HUF_REPEAT_VALID {
6
} else {
COMPRESS_LITERALS_SIZE_MIN
};
if src_size <= min_lit_size {
unsafe { (*huf_metadata).hType = SET_BASIC };
return 0;
}
}
/* Scan input and build symbol stats */
{
let largest = unsafe {
HIST_count_wksp(
count_wksp,
&mut max_symbol_value,
src,
src_size,
workspace,
wksp_size,
)
};
if ERR_isError(largest) {
return largest;
}
if largest == src_size {
/* only one literal symbol */
unsafe { (*huf_metadata).hType = SET_RLE };
return 0;
}
if largest <= (src_size >> 7) + 4 {
/* heuristic: likely not compressible */
unsafe { (*huf_metadata).hType = SET_BASIC };
return 0;
}
}
/* Validate the previous Huffman table */
if repeat == HUF_REPEAT_CHECK
&& unsafe {
HUF_validateCTable(
ptr::addr_of!((*prev_huf).CTable).cast::<usize>(),
count_wksp,
max_symbol_value,
)
} == 0
{
repeat = HUF_REPEAT_NONE;
}
/* Build Huffman Tree */
unsafe {
ptr::write_bytes(
ptr::addr_of_mut!((*next_huf).CTable).cast::<u8>(),
0,
size_of_val(&(*next_huf).CTable),
);
}
huff_log = unsafe {
HUF_optimalTableLog(
huff_log,
src_size,
max_symbol_value,
node_wksp,
node_wksp_size,
ptr::addr_of_mut!((*next_huf).CTable).cast::<usize>(),
count_wksp,
huf_flags,
)
};
debug_assert!(huff_log <= LIT_HUF_LOG);
{
let max_bits = unsafe {
HUF_buildCTable_wksp(
ptr::addr_of_mut!((*next_huf).CTable).cast::<usize>(),
count_wksp,
max_symbol_value,
huff_log,
node_wksp,
node_wksp_size,
)
};
if ERR_isError(max_bits) {
return max_bits;
}
huff_log = max_bits as c_uint;
}
{
/* Build and write the CTable */
let new_c_size = unsafe {
HUF_estimateCompressedSize(
ptr::addr_of!((*next_huf).CTable).cast::<usize>(),
count_wksp,
max_symbol_value,
)
};
let h_size = unsafe {
HUF_writeCTable_wksp(
ptr::addr_of_mut!((*huf_metadata).hufDesBuffer).cast::<c_void>(),
size_of::<[u8; ZSTD_MAX_HUF_HEADER_SIZE]>(),
ptr::addr_of!((*next_huf).CTable).cast::<usize>(),
max_symbol_value,
huff_log,
node_wksp,
node_wksp_size,
)
};
/* Check against repeating the previous CTable */
if repeat != HUF_REPEAT_NONE {
let old_c_size = unsafe {
HUF_estimateCompressedSize(
ptr::addr_of!((*prev_huf).CTable).cast::<usize>(),
count_wksp,
max_symbol_value,
)
};
if old_c_size < src_size
&& (old_c_size <= h_size.wrapping_add(new_c_size)
|| h_size.wrapping_add(12) >= src_size)
{
unsafe {
copy_huf_tables(next_huf, prev_huf);
(*huf_metadata).hType = SET_REPEAT;
}
return 0;
}
}
if new_c_size.wrapping_add(h_size) >= src_size {
unsafe {
copy_huf_tables(next_huf, prev_huf);
(*huf_metadata).hType = SET_BASIC;
}
return 0;
}
unsafe {
(*huf_metadata).hType = SET_COMPRESSED;
(*next_huf).repeatMode = HUF_REPEAT_CHECK;
}
h_size
}
}
/// C's `ZSTD_buildDummySequencesStatistics()`: marks every encoding type as
/// basic and resets the FSE repeat modes.
unsafe fn build_dummy_sequences_statistics(
next_entropy: *mut ZSTD_fseCTables_t,
) -> SymbolEncodingTypeStats {
unsafe {
(*next_entropy).litlength_repeatMode = FSE_REPEAT_NONE;
(*next_entropy).offcode_repeatMode = FSE_REPEAT_NONE;
(*next_entropy).matchlength_repeatMode = FSE_REPEAT_NONE;
}
SymbolEncodingTypeStats {
ll_type: SET_BASIC as u32,
off_type: SET_BASIC as u32,
ml_type: SET_BASIC as u32,
size: 0,
last_count_size: 0,
long_offsets: 0,
}
}
/// C's `ZSTD_buildBlockEntropyStats_sequences()`.
#[allow(clippy::too_many_arguments)]
unsafe fn build_block_entropy_stats_sequences(
seq_store: *const SeqStore_t,
prev_entropy: *const ZSTD_fseCTables_t,
next_entropy: *mut ZSTD_fseCTables_t,
strategy: c_int,
fse_metadata: *mut ZSTD_fseCTablesMetadata_t,
workspace: *mut c_void,
wksp_size: usize,
) -> usize {
let store = unsafe { &*seq_store };
let nb_seq = unsafe { store.sequences.offset_from(store.sequencesStart) } as usize;
let count_workspace = workspace.cast::<c_uint>();
let entropy_workspace = unsafe { count_workspace.add(MAX_SEQ + 1) }.cast::<c_void>();
let entropy_workspace_size = wksp_size.wrapping_sub((MAX_SEQ + 1) * size_of::<c_uint>());
let stats = if nb_seq != 0 {
unsafe {
build_sequences_statistics(
seq_store,
nb_seq,
prev_entropy,
next_entropy,
ptr::addr_of_mut!((*fse_metadata).fseTablesBuffer).cast::<u8>(),
ZSTD_MAX_FSE_HEADERS_SIZE,
strategy,
count_workspace,
entropy_workspace,
entropy_workspace_size,
)
}
} else {
unsafe { build_dummy_sequences_statistics(next_entropy) }
};
if ERR_isError(stats.size) {
return stats.size;
}
unsafe {
(*fse_metadata).llType = stats.ll_type as c_int;
(*fse_metadata).ofType = stats.off_type as c_int;
(*fse_metadata).mlType = stats.ml_type as c_int;
(*fse_metadata).lastCountSize = stats.last_count_size;
}
stats.size
}
/// C's `ZSTD_buildBlockEntropyStats()`: builds entropy statistics for one
/// block. Requires an `ENTROPY_WORKSPACE_SIZE` workspace.
#[allow(clippy::too_many_arguments)]
pub(crate) unsafe fn build_block_entropy_stats(
seq_store: *const SeqStore_t,
prev_entropy: *const ZSTD_entropyCTables_t,
next_entropy: *mut ZSTD_entropyCTables_t,
strategy: c_int,
disable_literal_compression: c_int,
entropy_metadata: *mut ZSTD_entropyCTablesMetadata_t,
workspace: *mut c_void,
wksp_size: usize,
) -> usize {
let store = unsafe { &*seq_store };
let lit_size = unsafe { store.lit.offset_from(store.litStart) } as usize;
let huf_flags = if strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD {
HUF_FLAGS_OPTIMAL_DEPTH
} else {
0
};
let huf_des_size = unsafe {
build_block_entropy_stats_literals(
store.litStart.cast::<c_void>(),
lit_size,
ptr::addr_of!((*prev_entropy).huf),
ptr::addr_of_mut!((*next_entropy).huf),
ptr::addr_of_mut!((*entropy_metadata).hufMetadata),
disable_literal_compression,
workspace,
wksp_size,
huf_flags,
)
};
unsafe { (*entropy_metadata).hufMetadata.hufDesSize = huf_des_size };
if ERR_isError(huf_des_size) {
return huf_des_size;
}
let fse_tables_size = unsafe {
build_block_entropy_stats_sequences(
seq_store,
ptr::addr_of!((*prev_entropy).fse),
ptr::addr_of_mut!((*next_entropy).fse),
strategy,
ptr::addr_of_mut!((*entropy_metadata).fseMetadata),
workspace,
wksp_size,
)
};
unsafe { (*entropy_metadata).fseMetadata.fseTablesSize = fse_tables_size };
if ERR_isError(fse_tables_size) {
return fse_tables_size;
}
0
}
/// C ABI wrapper behind the `ZSTD_buildBlockEntropyStats()` shim, which is
/// still exported from C for the block splitter's size estimation.
#[allow(clippy::too_many_arguments)]
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_buildBlockEntropyStats(
seq_store: *const SeqStore_t,
prev_entropy: *const ZSTD_entropyCTables_t,
next_entropy: *mut ZSTD_entropyCTables_t,
strategy: c_int,
disable_literal_compression: c_int,
entropy_metadata: *mut ZSTD_entropyCTablesMetadata_t,
workspace: *mut c_void,
wksp_size: usize,
) -> usize {
unsafe {
build_block_entropy_stats(
seq_store,
prev_entropy,
next_entropy,
strategy,
disable_literal_compression,
entropy_metadata,
workspace,
wksp_size,
)
}
}
/// C's `ZSTD_copyBlockSequences()`: exports a block's sequences in the public
/// `ZSTD_Sequence` format, appending the last-literals block delimiter.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_copyBlockSequences(
seq_collector: *mut SeqCollector,
seq_store: *const SeqStore_t,
prev_repcodes: *const u32,
) -> usize {
let collector = unsafe { &mut *seq_collector };
let store = unsafe { &*seq_store };
let in_seqs = store.sequencesStart;
let nb_in_sequences = unsafe { store.sequences.offset_from(in_seqs) } as usize;
let nb_in_literals = unsafe { store.lit.offset_from(store.litStart) } as usize;
let out_seqs = if collector.seqIndex == 0 {
collector.seqStart
} else {
unsafe { collector.seqStart.add(collector.seqIndex) }
};
let nb_out_sequences = nb_in_sequences + 1;
let mut nb_out_literals = 0usize;
/* Bounds check that we have enough space for every input sequence
* and the block delimiter
*/
debug_assert!(collector.seqIndex <= collector.maxSequences);
if nb_out_sequences > collector.maxSequences.wrapping_sub(collector.seqIndex) {
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
let mut repcodes = [0u32; ZSTD_REP_NUM];
unsafe { ptr::copy_nonoverlapping(prev_repcodes, repcodes.as_mut_ptr(), ZSTD_REP_NUM) };
for index in 0..nb_in_sequences {
let in_seq = unsafe { in_seqs.add(index).read() };
let out = unsafe { out_seqs.add(index) };
let mut lit_length = in_seq.litLength as u32;
let mut match_length = in_seq.mlBase as u32 + MINMATCH as u32;
let mut rep = 0u32;
/* Handle the possible single length >= 64K
* There can only be one because we add MINMATCH to every match length,
* and blocks are at most 128K.
*/
if index == store.longLengthPos as usize {
if store.longLengthType == ZSTD_LLT_LITERAL_LENGTH {
lit_length += 0x10000;
} else if store.longLengthType == ZSTD_LLT_MATCH_LENGTH {
match_length += 0x10000;
}
}
/* Determine the raw offset given the offBase, which may be a repcode. */
let raw_offset = if (1..=ZSTD_REP_NUM as u32).contains(&in_seq.offBase) {
let repcode = in_seq.offBase;
debug_assert!(repcode > 0);
rep = repcode;
if lit_length != 0 {
repcodes[repcode as usize - 1]
} else if repcode == 3 {
debug_assert!(repcodes[0] > 1);
repcodes[0].wrapping_sub(1)
} else {
repcodes[repcode as usize]
}
} else {
in_seq.offBase.wrapping_sub(ZSTD_REP_NUM as u32)
};
unsafe {
(*out).litLength = lit_length;
(*out).matchLength = match_length;
(*out).offset = raw_offset;
(*out).rep = rep;
}
/* Update repcode history for the sequence */
update_rep(&mut repcodes, in_seq.offBase, in_seq.litLength == 0);
nb_out_literals += lit_length as usize;
}
/* Insert last literals (if any exist) in the block as a sequence with ml == off == 0.
* If there are no last literals, then we'll emit (of: 0, ml: 0, ll: 0), which is a marker
* for the block boundary, according to the API.
*/
debug_assert!(nb_in_literals >= nb_out_literals);
{
let last_ll_size = nb_in_literals.wrapping_sub(nb_out_literals);
unsafe {
let last = out_seqs.add(nb_in_sequences);
(*last).litLength = last_ll_size as u32;
(*last).matchLength = 0;
(*last).offset = 0;
}
}
collector.seqIndex += nb_out_sequences;
debug_assert!(collector.seqIndex <= collector.maxSequences);
0
}
#[cfg(test)]
mod tests {
use super::*;
use std::mem::{align_of, offset_of};
const HUF_CTABLE_SIZE_ST: usize = HUF_SYMBOLVALUE_MAX as usize + 2;
const ENTROPY_WORKSPACE_SIZE: usize = (8 << 10) + 512 + (MAX_SEQ + 2) * size_of::<u32>();
fn empty_huf_tables(repeat: c_int) -> ZSTD_hufCTables_t {
ZSTD_hufCTables_t {
CTable: [0; HUF_CTABLE_SIZE_ST],
repeatMode: repeat,
}
}
fn empty_fse_tables(repeat: c_int) -> ZSTD_fseCTables_t {
ZSTD_fseCTables_t {
offcodeCTable: [0; OFF_CTABLE_SIZE],
matchlengthCTable: [0; ML_CTABLE_SIZE],
litlengthCTable: [0; LL_CTABLE_SIZE],
offcode_repeatMode: repeat,
matchlength_repeatMode: repeat,
litlength_repeatMode: repeat,
}
}
struct TestSeqStore {
sequences: Vec<SeqDef>,
literals: Vec<u8>,
ll_codes: Vec<u8>,
ml_codes: Vec<u8>,
of_codes: Vec<u8>,
}
impl TestSeqStore {
fn new(sequences: Vec<SeqDef>, literals: Vec<u8>) -> Self {
let nb_seq = sequences.len();
TestSeqStore {
sequences,
literals,
ll_codes: vec![0; nb_seq],
ml_codes: vec![0; nb_seq],
of_codes: vec![0; nb_seq],
}
}
fn seq_store(&mut self) -> SeqStore_t {
let nb_seq = self.sequences.len();
let nb_lit = self.literals.len();
SeqStore_t {
sequencesStart: self.sequences.as_mut_ptr(),
sequences: unsafe { self.sequences.as_mut_ptr().add(nb_seq) },
litStart: self.literals.as_mut_ptr(),
lit: unsafe { self.literals.as_mut_ptr().add(nb_lit) },
llCode: self.ll_codes.as_mut_ptr(),
mlCode: self.ml_codes.as_mut_ptr(),
ofCode: self.of_codes.as_mut_ptr(),
maxNbSeq: nb_seq,
maxNbLit: nb_lit,
longLengthType: 0,
longLengthPos: 0,
}
}
}
#[test]
fn c_leaf_layouts_match_supported_abis() {
assert_eq!(size_of::<SeqDef>(), 8);
assert_eq!(align_of::<SeqDef>(), align_of::<u32>());
assert_eq!(offset_of!(SeqStore_t, sequencesStart), 0);
assert_eq!(
offset_of!(SeqStore_t, longLengthPos),
9 * size_of::<usize>() + 4
);
assert_eq!(size_of::<SeqStore_t>(), 9 * size_of::<usize>() + 8);
assert_eq!(size_of::<ZSTD_fseCTables_t>(), 3552);
assert_eq!(offset_of!(ZSTD_compressedBlockState_t, entropy), 0);
assert_eq!(
offset_of!(ZSTD_compressedBlockState_t, rep),
size_of::<ZSTD_entropyCTables_t>()
);
assert_eq!(size_of::<ZSTD_Sequence>(), 16);
assert_eq!(offset_of!(SeqCollector, seqStart), size_of::<usize>());
assert_eq!(offset_of!(SeqCollector, seqIndex), 2 * size_of::<usize>());
assert_eq!(size_of::<SeqCollector>(), 4 * size_of::<usize>());
if size_of::<usize>() == 8 {
assert_eq!(size_of::<ZSTD_hufCTablesMetadata_t>(), 144);
assert_eq!(size_of::<ZSTD_fseCTablesMetadata_t>(), 168);
assert_eq!(size_of::<ZSTD_entropyCTablesMetadata_t>(), 312);
assert_eq!(size_of::<ZSTD_entropyCTables_t>(), 5616);
assert_eq!(size_of::<ZSTD_compressedBlockState_t>(), 5632);
} else {
assert_eq!(size_of::<ZSTD_hufCTablesMetadata_t>(), 136);
assert_eq!(size_of::<ZSTD_fseCTablesMetadata_t>(), 156);
assert_eq!(size_of::<ZSTD_entropyCTablesMetadata_t>(), 292);
assert_eq!(size_of::<ZSTD_entropyCTables_t>(), 4584);
assert_eq!(size_of::<ZSTD_compressedBlockState_t>(), 4596);
}
}
#[test]
fn repcode_updates_match_the_c_sum_type_rules() {
let mut reps = [1, 4, 8];
update_rep(&mut reps, 10, false);
assert_eq!(reps, [7, 1, 4]);
update_rep(&mut reps, 1, true);
assert_eq!(reps, [1, 7, 4]);
update_rep(&mut reps, 2, false);
assert_eq!(reps, [7, 1, 4]);
update_rep(&mut reps, 3, false);
assert_eq!(reps, [4, 7, 1]);
}
#[test]
fn seq_to_codes_matches_the_reference_tables() {
let mut store = TestSeqStore::new(
vec![
SeqDef {
offBase: 4,
litLength: 0,
mlBase: 0,
},
SeqDef {
offBase: 1,
litLength: 63,
mlBase: 127,
},
SeqDef {
offBase: 1 << 20,
litLength: 64,
mlBase: 128,
},
],
vec![0; 127],
);
let seq_store = store.seq_store();
let long_offsets = unsafe { ZSTD_seqToCodes(&seq_store) };
assert_eq!(long_offsets, 0);
assert_eq!(store.ll_codes, [0, 24, 25]);
assert_eq!(store.ml_codes, [0, 42, 43]);
assert_eq!(store.of_codes, [2, 0, 20]);
}
#[test]
fn rle_sequences_produce_one_byte_table_headers() {
// Five identical sequences select set_rle for all three symbol kinds;
// each table header is then exactly the one RLE symbol byte.
let mut store = TestSeqStore::new(
vec![
SeqDef {
offBase: 8,
litLength: 2,
mlBase: 3,
};
5
],
vec![b'x'; 10],
);
let seq_store = store.seq_store();
let prev = empty_fse_tables(FSE_REPEAT_NONE);
let mut next = empty_fse_tables(FSE_REPEAT_NONE);
let mut metadata =
unsafe { std::mem::MaybeUninit::<ZSTD_fseCTablesMetadata_t>::zeroed().assume_init() };
let mut workspace = vec![0u64; ENTROPY_WORKSPACE_SIZE / size_of::<u64>()];
let size = unsafe {
build_block_entropy_stats_sequences(
&seq_store,
&prev,
&mut next,
1,
&mut metadata,
workspace.as_mut_ptr().cast::<c_void>(),
ENTROPY_WORKSPACE_SIZE,
)
};
assert_eq!(size, 3);
assert_eq!(metadata.llType, SET_RLE);
assert_eq!(metadata.ofType, SET_RLE);
assert_eq!(metadata.mlType, SET_RLE);
assert_eq!(metadata.lastCountSize, 0);
assert_eq!(&metadata.fseTablesBuffer[..3], &[2, 3, 3]);
assert_eq!(next.litlength_repeatMode, FSE_REPEAT_NONE);
assert_eq!(next.offcode_repeatMode, FSE_REPEAT_NONE);
assert_eq!(next.matchlength_repeatMode, FSE_REPEAT_NONE);
}
#[test]
fn empty_seq_store_emits_raw_literals_and_repeats_tables() {
let mut store = TestSeqStore::new(Vec::new(), b"hello".to_vec());
let seq_store = store.seq_store();
let prev = ZSTD_entropyCTables_t {
huf: empty_huf_tables(HUF_REPEAT_NONE),
fse: empty_fse_tables(2),
};
let mut next = ZSTD_entropyCTables_t {
huf: empty_huf_tables(HUF_REPEAT_NONE),
fse: empty_fse_tables(FSE_REPEAT_NONE),
};
let mut workspace = vec![0u64; ENTROPY_WORKSPACE_SIZE / size_of::<u64>()];
let mut output = [0xA5u8; 32];
let size = unsafe {
ZSTD_rust_entropyCompressSeqStore_internal(
output.as_mut_ptr().cast::<c_void>(),
output.len(),
seq_store.litStart.cast::<c_void>(),
5,
&seq_store,
&prev,
&mut next,
1,
0,
workspace.as_mut_ptr().cast::<c_void>(),
ENTROPY_WORKSPACE_SIZE,
0,
)
};
// Raw literals header (5 << 3), five literal bytes, nbSeq == 0.
assert_eq!(size, 7);
assert_eq!(&output[..7], &[0x28, b'h', b'e', b'l', b'l', b'o', 0x00]);
// With no sequences, the previous FSE tables are copied unchanged.
assert_eq!(next.fse.offcode_repeatMode, 2);
assert_eq!(next.fse.matchlength_repeatMode, 2);
assert_eq!(next.fse.litlength_repeatMode, 2);
}
#[test]
fn literal_stats_pick_rle_and_compressed_types() {
let mut workspace = vec![0u64; ENTROPY_WORKSPACE_SIZE / size_of::<u64>()];
let prev = empty_huf_tables(HUF_REPEAT_NONE);
let mut next = empty_huf_tables(HUF_REPEAT_NONE);
let mut metadata =
unsafe { std::mem::MaybeUninit::<ZSTD_hufCTablesMetadata_t>::zeroed().assume_init() };
let rle = [7u8; 128];
let size = unsafe {
build_block_entropy_stats_literals(
rle.as_ptr().cast::<c_void>(),
rle.len(),
&prev,
&mut next,
&mut metadata,
0,
workspace.as_mut_ptr().cast::<c_void>(),
ENTROPY_WORKSPACE_SIZE,
0,
)
};
assert_eq!(size, 0);
assert_eq!(metadata.hType, SET_RLE);
let mut compressible = [0u8; 1024];
for (index, byte) in compressible.iter_mut().enumerate() {
*byte = (index % 4) as u8;
}
let size = unsafe {
build_block_entropy_stats_literals(
compressible.as_ptr().cast::<c_void>(),
compressible.len(),
&prev,
&mut next,
&mut metadata,
0,
workspace.as_mut_ptr().cast::<c_void>(),
ENTROPY_WORKSPACE_SIZE,
0,
)
};
assert!(!ERR_isError(size));
assert!(size > 0);
assert_eq!(metadata.hType, SET_COMPRESSED);
assert_eq!(metadata.hufDesSize, 0); // caller stores the returned size
assert_eq!(next.repeatMode, HUF_REPEAT_CHECK);
}
#[test]
fn copied_sequences_resolve_repcodes_and_append_the_delimiter() {
let mut store = TestSeqStore::new(
vec![
SeqDef {
offBase: 3 + 100, // raw offset 100
litLength: 4,
mlBase: 5,
},
SeqDef {
offBase: 1, // repcode 1 with literals: rep[0] == 100
litLength: 2,
mlBase: 7,
},
SeqDef {
offBase: 3, // repcode 3 without literals: rep[0] - 1 == 99
litLength: 0,
mlBase: 9,
},
],
vec![0; 10],
);
let seq_store = store.seq_store();
let mut out = [ZSTD_Sequence {
offset: 0xAA,
litLength: 0xAA,
matchLength: 0xAA,
rep: 0xAA,
}; 8];
let mut collector = SeqCollector {
collectSequences: 1,
seqStart: out.as_mut_ptr(),
seqIndex: 1,
maxSequences: 8,
};
let reps = [1u32, 4, 8];
let result =
unsafe { ZSTD_rust_copyBlockSequences(&mut collector, &seq_store, reps.as_ptr()) };
assert_eq!(result, 0);
assert_eq!(collector.seqIndex, 5);
let expect = [
(100, 4, 5 + 3, 0),
(100, 2, 7 + 3, 1),
(99, 0, 9 + 3, 3),
(0, 10 - 6, 0, 0xAA), // delimiter: last literals; rep untouched
];
for (index, &(offset, lit, matched, rep)) in expect.iter().enumerate() {
assert_eq!(out[1 + index].offset, offset, "offset {index}");
assert_eq!(out[1 + index].litLength, lit, "litLength {index}");
assert_eq!(out[1 + index].matchLength, matched, "matchLength {index}");
assert_eq!(out[1 + index].rep, rep, "rep {index}");
}
collector.maxSequences = 5;
let result =
unsafe { ZSTD_rust_copyBlockSequences(&mut collector, &seq_store, reps.as_ptr()) };
assert_eq!(result, ERROR(ZstdErrorCode::DstSizeTooSmall));
}
}
rust/src/zstd_compress_superblock.rs
+49 -191
View File
@@ -6,13 +6,13 @@
//! This is the Rust implementation of `zstd_compress_superblock.c`. Its C
//! entry-point shim only extracts fields from the opaque `ZSTD_CCtx`; all
//! sequence partitioning, literal and sequence section writing, entropy
//! fallback, and repcode repair remain here. The leaf layouts below are
//! intentionally kept C-shaped and checked for both supported pointer widths.
//! fallback, and repcode repair remain here. The C-shaped leaf layouts are
//! shared with `zstd_compress_stats`, which also owns the block entropy
//! statistics this module consumes.
use crate::common::{
LL_BITS, LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, MAX_LL, MAX_ML, MAX_OFF, MINMATCH, ML_BITS,
ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG,
ZSTD_MAX_FSE_HEADERS_SIZE, ZSTD_MAX_HUF_HEADER_SIZE,
};
use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
use crate::hist::{HIST_countFast_wksp, HIST_count_wksp};
@@ -23,9 +23,17 @@ use crate::mem::{MEM_32bits, MEM_writeLE16, MEM_writeLE24, MEM_writeLE32};
use crate::zstd_compress_literals::{
ZSTD_compressRleLiteralsBlock, ZSTD_hufCTables_t, ZSTD_noCompressLiterals,
};
use crate::zstd_compress_sequences::{
SeqDef, ZSTD_crossEntropyCost, ZSTD_encodeSequences, ZSTD_fseBitCost,
};
use crate::zstd_compress_stats::{
build_block_entropy_stats, update_rep, SeqStore_t, ZSTD_compressedBlockState_t,
ZSTD_entropyCTablesMetadata_t, ZSTD_entropyCTables_t, ZSTD_fseCTablesMetadata_t,
ZSTD_fseCTables_t, ZSTD_hufCTablesMetadata_t,
};
use std::ffi::c_void;
use std::mem::MaybeUninit;
use std::os::raw::{c_int, c_short, c_uint};
use std::os::raw::c_int;
use std::ptr;
const SET_BASIC: c_int = 0;
@@ -42,121 +50,6 @@ const LONG_NB_SEQ: usize = 0x7f00;
const DEFAULT_MAX_OFF: u32 = 28;
const BYTE_SCALE: usize = 256;
const OFF_CTABLE_SIZE: usize = 1 + (1 << 7) + ((MAX_OFF + 1) * 2);
const ML_CTABLE_SIZE: usize = 1 + (1 << 8) + ((MAX_ML + 1) * 2);
const LL_CTABLE_SIZE: usize = 1 + (1 << 8) + ((MAX_LL + 1) * 2);
/// ABI-compatible `SeqDef` from `zstd_compress_internal.h`.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default)]
struct SeqDef {
offBase: u32,
litLength: u16,
mlBase: u16,
}
/// ABI-compatible `SeqStore_t` leaf layout.
///
/// The C context itself remains opaque. The C shim passes its `seqStore`
/// member directly, so this small, stable hot-path structure is the only
/// sequence storage representation crossing into Rust.
#[repr(C)]
struct SeqStore_t {
sequencesStart: *mut SeqDef,
sequences: *mut SeqDef,
litStart: *mut u8,
lit: *mut u8,
llCode: *mut u8,
mlCode: *mut u8,
ofCode: *mut u8,
maxNbSeq: usize,
maxNbLit: usize,
longLengthType: c_int,
longLengthPos: u32,
}
/// C's `ZSTD_fseCTables_t`. The table element type is `FSE_CTable`, an
/// `unsigned`, and the table lengths are the header macros expanded above.
#[repr(C)]
struct ZSTD_fseCTables_t {
offcodeCTable: [u32; OFF_CTABLE_SIZE],
matchlengthCTable: [u32; ML_CTABLE_SIZE],
litlengthCTable: [u32; LL_CTABLE_SIZE],
offcode_repeatMode: c_int,
matchlength_repeatMode: c_int,
litlength_repeatMode: c_int,
}
#[repr(C)]
struct ZSTD_entropyCTables_t {
huf: ZSTD_hufCTables_t,
fse: ZSTD_fseCTables_t,
}
#[repr(C)]
struct ZSTD_compressedBlockState_t {
entropy: ZSTD_entropyCTables_t,
rep: [u32; 3],
}
#[repr(C)]
struct ZSTD_hufCTablesMetadata_t {
hType: c_int,
hufDesBuffer: [u8; ZSTD_MAX_HUF_HEADER_SIZE],
hufDesSize: usize,
}
#[repr(C)]
struct ZSTD_fseCTablesMetadata_t {
llType: c_int,
ofType: c_int,
mlType: c_int,
fseTablesBuffer: [u8; ZSTD_MAX_FSE_HEADERS_SIZE],
fseTablesSize: usize,
lastCountSize: usize,
}
#[repr(C)]
struct ZSTD_entropyCTablesMetadata_t {
hufMetadata: ZSTD_hufCTablesMetadata_t,
fseMetadata: ZSTD_fseCTablesMetadata_t,
}
unsafe extern "C" {
fn ZSTD_buildBlockEntropyStats(
seq_store: *const SeqStore_t,
prev_entropy: *const ZSTD_entropyCTables_t,
next_entropy: *mut ZSTD_entropyCTables_t,
cctx_params: *const c_void,
entropy_metadata: *mut ZSTD_entropyCTablesMetadata_t,
workspace: *mut c_void,
wksp_size: usize,
) -> usize;
fn ZSTD_encodeSequences(
dst: *mut c_void,
dst_capacity: usize,
ctable_match_length: *const u32,
ml_code_table: *const u8,
ctable_offset_bits: *const u32,
of_code_table: *const u8,
ctable_lit_length: *const u32,
ll_code_table: *const u8,
sequences: *const SeqDef,
nb_seq: usize,
long_offsets: c_int,
bmi2: c_int,
) -> usize;
fn ZSTD_fseBitCost(ctable: *const u32, count: *const u32, max: c_uint) -> usize;
fn ZSTD_crossEntropyCost(
norm: *const c_short,
accuracy_log: c_uint,
count: *const c_uint,
max: c_uint,
) -> usize;
}
#[inline]
fn remaining_capacity(capacity: usize, written: usize) -> Result<usize, usize> {
capacity
@@ -838,28 +731,6 @@ unsafe fn size_block_sequences(
nb_seq
}
#[inline]
fn update_rep(reps: &mut [u32; 3], off_base: u32, literal_length_is_zero: bool) {
if off_base > 3 {
reps[2] = reps[1];
reps[1] = reps[0];
reps[0] = off_base - 3;
return;
}
let rep_code = off_base - 1 + u32::from(literal_length_is_zero);
if rep_code == 0 {
return;
}
let current_offset = if rep_code == 3 {
reps[0].wrapping_sub(1)
} else {
reps[rep_code as usize]
};
reps[2] = if rep_code >= 2 { reps[1] } else { reps[2] };
reps[1] = reps[0];
reps[0] = current_offset;
}
#[allow(clippy::too_many_arguments, clippy::manual_checked_ops)]
unsafe fn compress_subblock_multi(
seq_store: *const SeqStore_t,
@@ -1098,16 +969,18 @@ unsafe fn compress_subblock_multi(
/// C ABI implementation called by the declaration-only C superblock shim.
///
/// `cctx_params` deliberately remains opaque: only its two required scalar
/// fields are read by the shim, while the existing C entropy builder receives
/// the original pointer unchanged.
/// `ZSTD_CCtx_params` deliberately remains opaque: the C shim extracts the
/// two scalar fields this path reads (the strategy and the
/// literals-compression switch) and the crate-internal entropy statistics
/// builder receives them directly.
#[allow(clippy::too_many_arguments)]
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_compressSuperBlock(
seq_store: *const c_void,
prev_cblock: *const c_void,
next_cblock: *mut c_void,
cctx_params: *const c_void,
strategy: c_int,
disable_literal_compression: c_int,
workspace: *mut c_void,
wksp_size: usize,
bmi2: c_int,
@@ -1122,16 +995,17 @@ pub unsafe extern "C" fn ZSTD_rust_compressSuperBlock(
let seq_store = seq_store.cast::<SeqStore_t>();
let prev_cblock = prev_cblock.cast::<ZSTD_compressedBlockState_t>();
let next_cblock = next_cblock.cast::<ZSTD_compressedBlockState_t>();
// C only writes the used prefixes of its two metadata byte buffers. Start
// with initialized storage so treating the completed C struct as a Rust
// value never exposes uninitialized array elements.
// The statistics builder only writes the used prefixes of the two
// metadata byte buffers. Start with initialized storage so treating the
// completed struct as a Rust value never exposes uninitialized elements.
let mut entropy_metadata = MaybeUninit::<ZSTD_entropyCTablesMetadata_t>::zeroed();
let entropy_result = unsafe {
ZSTD_buildBlockEntropyStats(
build_block_entropy_stats(
seq_store,
ptr::addr_of!((*prev_cblock).entropy),
ptr::addr_of_mut!((*next_cblock).entropy),
cctx_params,
strategy,
disable_literal_compression,
entropy_metadata.as_mut_ptr(),
workspace,
wksp_size,
@@ -1163,50 +1037,34 @@ pub unsafe extern "C" fn ZSTD_rust_compressSuperBlock(
#[cfg(test)]
mod tests {
// The shared C leaf layouts and the repcode update rules are asserted in
// `zstd_compress_stats`, which owns those definitions.
use super::*;
use std::mem::{align_of, offset_of, size_of};
#[test]
fn c_leaf_layouts_match_supported_abis() {
assert_eq!(size_of::<SeqDef>(), 8);
assert_eq!(align_of::<SeqDef>(), align_of::<u32>());
assert_eq!(offset_of!(SeqStore_t, sequencesStart), 0);
assert_eq!(
offset_of!(SeqStore_t, longLengthPos),
9 * size_of::<usize>() + 4
);
assert_eq!(size_of::<SeqStore_t>(), 9 * size_of::<usize>() + 8);
assert_eq!(size_of::<ZSTD_fseCTables_t>(), 3552);
assert_eq!(offset_of!(ZSTD_compressedBlockState_t, entropy), 0);
assert_eq!(
offset_of!(ZSTD_compressedBlockState_t, rep),
size_of::<ZSTD_entropyCTables_t>()
);
if size_of::<usize>() == 8 {
assert_eq!(size_of::<ZSTD_hufCTablesMetadata_t>(), 144);
assert_eq!(size_of::<ZSTD_fseCTablesMetadata_t>(), 168);
assert_eq!(size_of::<ZSTD_entropyCTablesMetadata_t>(), 312);
assert_eq!(size_of::<ZSTD_entropyCTables_t>(), 5616);
assert_eq!(size_of::<ZSTD_compressedBlockState_t>(), 5632);
} else {
assert_eq!(size_of::<ZSTD_hufCTablesMetadata_t>(), 136);
assert_eq!(size_of::<ZSTD_fseCTablesMetadata_t>(), 156);
assert_eq!(size_of::<ZSTD_entropyCTablesMetadata_t>(), 292);
assert_eq!(size_of::<ZSTD_entropyCTables_t>(), 4584);
assert_eq!(size_of::<ZSTD_compressedBlockState_t>(), 4596);
}
}
fn subblock_sizing_respects_the_budget() {
let sequences = [SeqDef {
offBase: 4,
litLength: 10,
mlBase: 10,
}; 8];
#[test]
fn repcode_updates_match_the_c_sum_type_rules() {
let mut reps = [1, 4, 8];
update_rep(&mut reps, 10, false);
assert_eq!(reps, [7, 1, 4]);
update_rep(&mut reps, 1, true);
assert_eq!(reps, [1, 7, 4]);
update_rep(&mut reps, 2, false);
assert_eq!(reps, [7, 1, 4]);
update_rep(&mut reps, 3, false);
assert_eq!(reps, [4, 7, 1]);
// A generous budget keeps every sequence in one sub-block.
let all = unsafe {
size_block_sequences(
sequences.as_ptr(),
sequences.len(),
usize::MAX,
256,
256,
true,
)
};
assert_eq!(all, sequences.len());
// The first sequence always survives, even over budget.
let first =
unsafe { size_block_sequences(sequences.as_ptr(), sequences.len(), 0, 256, 256, true) };
assert_eq!(first, 1);
}
}