Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 ddidderr
|
039dec6c68
|
@@ -142,7 +142,6 @@ endif()
|
||||
set(_zstd_rust_features)
|
||||
set(_zstd_rust_compression 0)
|
||||
set(_zstd_rust_decompression 0)
|
||||
set(_zstd_rust_dictbuilder 0)
|
||||
if(ZSTD_BUILD_COMPRESSION)
|
||||
list(APPEND _zstd_rust_features compression)
|
||||
set(_zstd_rust_compression 1)
|
||||
@@ -151,10 +150,6 @@ if(ZSTD_BUILD_DECOMPRESSION)
|
||||
list(APPEND _zstd_rust_features decompression)
|
||||
set(_zstd_rust_decompression 1)
|
||||
endif()
|
||||
if(ZSTD_BUILD_DICTBUILDER)
|
||||
list(APPEND _zstd_rust_features dict-builder)
|
||||
set(_zstd_rust_dictbuilder 1)
|
||||
endif()
|
||||
|
||||
set(_zstd_rust_huf_mode default)
|
||||
if(_zstd_huf_force_x1)
|
||||
@@ -195,7 +190,7 @@ if(_zstd_rust_features)
|
||||
endif()
|
||||
|
||||
set(_zstd_rust_build_config
|
||||
"c${_zstd_rust_compression}-d${_zstd_rust_decompression}-b${_zstd_rust_dictbuilder}-${_zstd_rust_huf_mode}")
|
||||
"c${_zstd_rust_compression}-d${_zstd_rust_decompression}-${_zstd_rust_huf_mode}")
|
||||
set(ZSTD_RUST_MANIFEST "${ZSTD_SOURCE_DIR}/rust/Cargo.toml")
|
||||
set(ZSTD_RUST_TARGET_DIR
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/rust-target/${_zstd_rust_build_config}")
|
||||
|
||||
@@ -73,9 +73,7 @@ if rust_huf_force_x1 and rust_huf_force_x2
|
||||
error('HUF_FORCE_DECOMPRESS_X1 and HUF_FORCE_DECOMPRESS_X2 are mutually exclusive')
|
||||
endif
|
||||
|
||||
# Meson always compiles the dictBuilder sources above, so the Rust archive
|
||||
# must always carry the matching dict-builder module set.
|
||||
rust_features = ['compression', 'decompression', 'dict-builder']
|
||||
rust_features = ['compression', 'decompression']
|
||||
rust_huf_mode = 'default'
|
||||
rust_huf_c_args = []
|
||||
if rust_huf_force_x1
|
||||
@@ -98,7 +96,7 @@ if rust_target == ''
|
||||
endif
|
||||
endif
|
||||
|
||||
rust_build_config = 'c1-d1-b1-' + rust_huf_mode
|
||||
rust_build_config = 'c1-d1-' + rust_huf_mode
|
||||
rust_target_dir = join_paths(meson.current_build_dir(), 'rust-target', rust_build_config)
|
||||
is_msvc = cc_id == compiler_msvc or cc_id == 'clang-cl'
|
||||
rust_staticlib_name = is_msvc ? 'zstd_rs.lib' : 'libzstd_rs.a'
|
||||
|
||||
+3
-7
@@ -84,20 +84,16 @@ endif
|
||||
ifneq ($(ZSTD_LIB_DECOMPRESSION),0)
|
||||
RUST_CARGO_FEATURES += decompression
|
||||
endif
|
||||
ifneq ($(ZSTD_LIB_DICTBUILDER),0)
|
||||
RUST_CARGO_FEATURES += dict-builder
|
||||
endif
|
||||
|
||||
RUST_MODULE_CONFIG := c$(ZSTD_LIB_COMPRESSION)-d$(ZSTD_LIB_DECOMPRESSION)-b$(ZSTD_LIB_DICTBUILDER)
|
||||
RUST_HUF_FEATURE :=
|
||||
RUST_BUILD_CONFIG := $(RUST_MODULE_CONFIG)-default
|
||||
RUST_BUILD_CONFIG := c$(ZSTD_LIB_COMPRESSION)-d$(ZSTD_LIB_DECOMPRESSION)-default
|
||||
ifneq ($(RUST_HUF_FORCE_X1),0)
|
||||
RUST_HUF_FEATURE := huf-force-decompress-x1
|
||||
RUST_BUILD_CONFIG := $(RUST_MODULE_CONFIG)-huf-force-decompress-x1
|
||||
RUST_BUILD_CONFIG := c$(ZSTD_LIB_COMPRESSION)-d$(ZSTD_LIB_DECOMPRESSION)-huf-force-decompress-x1
|
||||
endif
|
||||
ifneq ($(RUST_HUF_FORCE_X2),0)
|
||||
RUST_HUF_FEATURE := huf-force-decompress-x2
|
||||
RUST_BUILD_CONFIG := $(RUST_MODULE_CONFIG)-huf-force-decompress-x2
|
||||
RUST_BUILD_CONFIG := c$(ZSTD_LIB_COMPRESSION)-d$(ZSTD_LIB_DECOMPRESSION)-huf-force-decompress-x2
|
||||
endif
|
||||
ifneq ($(RUST_HUF_FEATURE),)
|
||||
ifneq ($(ZSTD_LIB_DECOMPRESSION),0)
|
||||
|
||||
+88
-587
@@ -88,6 +88,67 @@ size_t ZSTD_rust_params_estimateCDictSizeFromCParams(
|
||||
size_t dictSize, ZSTD_compressionParameters cParams,
|
||||
int dictLoadMethod, const ZSTD_rustCDictSizing* sizing);
|
||||
|
||||
/* Sequence statistics and seqStore entropy compression live in Rust
|
||||
* (rust/src/zstd_compress_stats.rs), which also exports ZSTD_seqToCodes()
|
||||
* under its original name. The shims below extract the sequence store, the
|
||||
* entropy-table leaves, and the only two ZSTD_CCtx_params fields these paths
|
||||
* read (the strategy and the literals-compression switch), so the private
|
||||
* parameter structure layout never crosses the language boundary. The
|
||||
* shared leaf layouts are pinned by the compile-time asserts that follow. */
|
||||
size_t ZSTD_rust_entropyCompressSeqStore_internal(
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* literals, size_t litSize,
|
||||
const SeqStore_t* seqStorePtr,
|
||||
const ZSTD_entropyCTables_t* prevEntropy,
|
||||
ZSTD_entropyCTables_t* nextEntropy,
|
||||
int strategy, int disableLiteralCompression,
|
||||
void* entropyWorkspace, size_t entropyWkspSize,
|
||||
int bmi2);
|
||||
size_t ZSTD_rust_entropyCompressSeqStore(
|
||||
const SeqStore_t* seqStorePtr,
|
||||
const ZSTD_entropyCTables_t* prevEntropy,
|
||||
ZSTD_entropyCTables_t* nextEntropy,
|
||||
int strategy, int disableLiteralCompression,
|
||||
void* dst, size_t dstCapacity,
|
||||
size_t srcSize,
|
||||
void* entropyWorkspace, size_t entropyWkspSize,
|
||||
int bmi2);
|
||||
size_t ZSTD_rust_buildBlockEntropyStats(
|
||||
const SeqStore_t* seqStorePtr,
|
||||
const ZSTD_entropyCTables_t* prevEntropy,
|
||||
ZSTD_entropyCTables_t* nextEntropy,
|
||||
int strategy, int disableLiteralCompression,
|
||||
ZSTD_entropyCTablesMetadata_t* entropyMetadata,
|
||||
void* workspace, size_t wkspSize);
|
||||
size_t ZSTD_rust_copyBlockSequences(
|
||||
SeqCollector* seqCollector, const SeqStore_t* seqStore,
|
||||
const U32 prevRepcodes[ZSTD_REP_NUM]);
|
||||
|
||||
typedef char ZSTD_rust_stats_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1];
|
||||
typedef char ZSTD_rust_stats_seqstore_long_length_pos[
|
||||
(offsetof(SeqStore_t, longLengthPos) == 9 * sizeof(size_t) + 4) ? 1 : -1];
|
||||
typedef char ZSTD_rust_stats_seqstore_layout[
|
||||
(sizeof(SeqStore_t) == 9 * sizeof(size_t) + 8) ? 1 : -1];
|
||||
typedef char ZSTD_rust_stats_huf_layout[
|
||||
(sizeof(ZSTD_hufCTables_t) == 258 * sizeof(size_t)) ? 1 : -1];
|
||||
typedef char ZSTD_rust_stats_fse_layout[(sizeof(ZSTD_fseCTables_t) == 3552) ? 1 : -1];
|
||||
typedef char ZSTD_rust_stats_entropy_fse_offset[
|
||||
(offsetof(ZSTD_entropyCTables_t, fse) == sizeof(ZSTD_hufCTables_t)) ? 1 : -1];
|
||||
typedef char ZSTD_rust_stats_huf_metadata_buffer[
|
||||
(offsetof(ZSTD_hufCTablesMetadata_t, hufDesBuffer) == 4
|
||||
&& ZSTD_MAX_HUF_HEADER_SIZE == 128) ? 1 : -1];
|
||||
typedef char ZSTD_rust_stats_fse_metadata_buffer[
|
||||
(offsetof(ZSTD_fseCTablesMetadata_t, fseTablesBuffer) == 12
|
||||
&& ZSTD_MAX_FSE_HEADERS_SIZE == 133) ? 1 : -1];
|
||||
typedef char ZSTD_rust_stats_entropy_metadata_layout[
|
||||
(offsetof(ZSTD_entropyCTablesMetadata_t, fseMetadata)
|
||||
== sizeof(ZSTD_hufCTablesMetadata_t)) ? 1 : -1];
|
||||
typedef char ZSTD_rust_stats_sequence_layout[(sizeof(ZSTD_Sequence) == 16) ? 1 : -1];
|
||||
typedef char ZSTD_rust_stats_seqcollector_layout[
|
||||
(offsetof(SeqCollector, seqStart) == sizeof(size_t)
|
||||
&& offsetof(SeqCollector, seqIndex) == 2 * sizeof(size_t)
|
||||
&& sizeof(SeqCollector) == 4 * sizeof(size_t)) ? 1 : -1];
|
||||
|
||||
#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
|
||||
# define ZSTD_RUST_ASAN_REDZONE_SIZE ((size_t)ZSTD_CWKSP_ASAN_REDZONE_SIZE)
|
||||
#else
|
||||
@@ -2561,33 +2622,7 @@ static void ZSTD_reduceIndex (ZSTD_MatchState_t* ms, ZSTD_CCtx_params const* par
|
||||
|
||||
/* See doc/zstd_compression_format.md for detailed format description */
|
||||
|
||||
int ZSTD_seqToCodes(const SeqStore_t* seqStorePtr)
|
||||
{
|
||||
const SeqDef* const sequences = seqStorePtr->sequencesStart;
|
||||
BYTE* const llCodeTable = seqStorePtr->llCode;
|
||||
BYTE* const ofCodeTable = seqStorePtr->ofCode;
|
||||
BYTE* const mlCodeTable = seqStorePtr->mlCode;
|
||||
U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
|
||||
U32 u;
|
||||
int longOffsets = 0;
|
||||
assert(nbSeq <= seqStorePtr->maxNbSeq);
|
||||
for (u=0; u<nbSeq; u++) {
|
||||
U32 const llv = sequences[u].litLength;
|
||||
U32 const ofCode = ZSTD_highbit32(sequences[u].offBase);
|
||||
U32 const mlv = sequences[u].mlBase;
|
||||
llCodeTable[u] = (BYTE)ZSTD_LLcode(llv);
|
||||
ofCodeTable[u] = (BYTE)ofCode;
|
||||
mlCodeTable[u] = (BYTE)ZSTD_MLcode(mlv);
|
||||
assert(!(MEM_64bits() && ofCode >= STREAM_ACCUMULATOR_MIN));
|
||||
if (MEM_32bits() && ofCode >= STREAM_ACCUMULATOR_MIN)
|
||||
longOffsets = 1;
|
||||
}
|
||||
if (seqStorePtr->longLengthType==ZSTD_llt_literalLength)
|
||||
llCodeTable[seqStorePtr->longLengthPos] = MaxLL;
|
||||
if (seqStorePtr->longLengthType==ZSTD_llt_matchLength)
|
||||
mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
|
||||
return longOffsets;
|
||||
}
|
||||
/* ZSTD_seqToCodes() lives in rust/src/zstd_compress_stats.rs. */
|
||||
|
||||
/* ZSTD_useTargetCBlockSize():
|
||||
* Returns if target compressed block size param is being used.
|
||||
@@ -2611,150 +2646,11 @@ static int ZSTD_blockSplitterEnabled(ZSTD_CCtx_params* cctxParams)
|
||||
return (cctxParams->postBlockSplitter == ZSTD_ps_enable);
|
||||
}
|
||||
|
||||
/* Type returned by ZSTD_buildSequencesStatistics containing finalized symbol encoding types
|
||||
* and size of the sequences statistics
|
||||
*/
|
||||
typedef struct {
|
||||
U32 LLtype;
|
||||
U32 Offtype;
|
||||
U32 MLtype;
|
||||
size_t size;
|
||||
size_t lastCountSize; /* Accounts for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */
|
||||
int longOffsets;
|
||||
} ZSTD_symbolEncodingTypeStats_t;
|
||||
/* ZSTD_buildSequencesStatistics() and the ZSTD_entropyCompressSeqStore*()
|
||||
* implementations live in rust/src/zstd_compress_stats.rs. The wrappers
|
||||
* below keep the original signatures and extract the only two
|
||||
* ZSTD_CCtx_params fields those paths read. */
|
||||
|
||||
/* ZSTD_buildSequencesStatistics():
|
||||
* Returns a ZSTD_symbolEncodingTypeStats_t, or a zstd error code in the `size` field.
|
||||
* Modifies `nextEntropy` to have the appropriate values as a side effect.
|
||||
* nbSeq must be greater than 0.
|
||||
*
|
||||
* entropyWkspSize must be of size at least ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1)*sizeof(U32)
|
||||
*/
|
||||
static ZSTD_symbolEncodingTypeStats_t
|
||||
ZSTD_buildSequencesStatistics(
|
||||
const SeqStore_t* seqStorePtr, size_t nbSeq,
|
||||
const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy,
|
||||
BYTE* dst, const BYTE* const dstEnd,
|
||||
ZSTD_strategy strategy, unsigned* countWorkspace,
|
||||
void* entropyWorkspace, size_t entropyWkspSize)
|
||||
{
|
||||
BYTE* const ostart = dst;
|
||||
const BYTE* const oend = dstEnd;
|
||||
BYTE* op = ostart;
|
||||
FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable;
|
||||
FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable;
|
||||
FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable;
|
||||
const BYTE* const ofCodeTable = seqStorePtr->ofCode;
|
||||
const BYTE* const llCodeTable = seqStorePtr->llCode;
|
||||
const BYTE* const mlCodeTable = seqStorePtr->mlCode;
|
||||
ZSTD_symbolEncodingTypeStats_t stats;
|
||||
|
||||
stats.lastCountSize = 0;
|
||||
/* convert length/distances into codes */
|
||||
stats.longOffsets = ZSTD_seqToCodes(seqStorePtr);
|
||||
assert(op <= oend);
|
||||
assert(nbSeq != 0); /* ZSTD_selectEncodingType() divides by nbSeq */
|
||||
/* build CTable for Literal Lengths */
|
||||
{ unsigned max = MaxLL;
|
||||
size_t const mostFrequent = HIST_countFast_wksp(countWorkspace, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */
|
||||
DEBUGLOG(5, "Building LL table");
|
||||
nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode;
|
||||
stats.LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode,
|
||||
countWorkspace, max, mostFrequent, nbSeq,
|
||||
LLFSELog, prevEntropy->litlengthCTable,
|
||||
LL_defaultNorm, LL_defaultNormLog,
|
||||
ZSTD_defaultAllowed, strategy);
|
||||
assert(set_basic < set_compressed && set_rle < set_compressed);
|
||||
assert(!(stats.LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
|
||||
{ size_t const countSize = ZSTD_buildCTable(
|
||||
op, (size_t)(oend - op),
|
||||
CTable_LitLength, LLFSELog, (SymbolEncodingType_e)stats.LLtype,
|
||||
countWorkspace, max, llCodeTable, nbSeq,
|
||||
LL_defaultNorm, LL_defaultNormLog, MaxLL,
|
||||
prevEntropy->litlengthCTable,
|
||||
sizeof(prevEntropy->litlengthCTable),
|
||||
entropyWorkspace, entropyWkspSize);
|
||||
if (ZSTD_isError(countSize)) {
|
||||
DEBUGLOG(3, "ZSTD_buildCTable for LitLens failed");
|
||||
stats.size = countSize;
|
||||
return stats;
|
||||
}
|
||||
if (stats.LLtype == set_compressed)
|
||||
stats.lastCountSize = countSize;
|
||||
op += countSize;
|
||||
assert(op <= oend);
|
||||
} }
|
||||
/* build CTable for Offsets */
|
||||
{ unsigned max = MaxOff;
|
||||
size_t const mostFrequent = HIST_countFast_wksp(
|
||||
countWorkspace, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */
|
||||
/* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */
|
||||
ZSTD_DefaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed;
|
||||
DEBUGLOG(5, "Building OF table");
|
||||
nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode;
|
||||
stats.Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode,
|
||||
countWorkspace, max, mostFrequent, nbSeq,
|
||||
OffFSELog, prevEntropy->offcodeCTable,
|
||||
OF_defaultNorm, OF_defaultNormLog,
|
||||
defaultPolicy, strategy);
|
||||
assert(!(stats.Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */
|
||||
{ size_t const countSize = ZSTD_buildCTable(
|
||||
op, (size_t)(oend - op),
|
||||
CTable_OffsetBits, OffFSELog, (SymbolEncodingType_e)stats.Offtype,
|
||||
countWorkspace, max, ofCodeTable, nbSeq,
|
||||
OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
|
||||
prevEntropy->offcodeCTable,
|
||||
sizeof(prevEntropy->offcodeCTable),
|
||||
entropyWorkspace, entropyWkspSize);
|
||||
if (ZSTD_isError(countSize)) {
|
||||
DEBUGLOG(3, "ZSTD_buildCTable for Offsets failed");
|
||||
stats.size = countSize;
|
||||
return stats;
|
||||
}
|
||||
if (stats.Offtype == set_compressed)
|
||||
stats.lastCountSize = countSize;
|
||||
op += countSize;
|
||||
assert(op <= oend);
|
||||
} }
|
||||
/* build CTable for MatchLengths */
|
||||
{ unsigned max = MaxML;
|
||||
size_t const mostFrequent = HIST_countFast_wksp(
|
||||
countWorkspace, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */
|
||||
DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op));
|
||||
nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode;
|
||||
stats.MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode,
|
||||
countWorkspace, max, mostFrequent, nbSeq,
|
||||
MLFSELog, prevEntropy->matchlengthCTable,
|
||||
ML_defaultNorm, ML_defaultNormLog,
|
||||
ZSTD_defaultAllowed, strategy);
|
||||
assert(!(stats.MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
|
||||
{ size_t const countSize = ZSTD_buildCTable(
|
||||
op, (size_t)(oend - op),
|
||||
CTable_MatchLength, MLFSELog, (SymbolEncodingType_e)stats.MLtype,
|
||||
countWorkspace, max, mlCodeTable, nbSeq,
|
||||
ML_defaultNorm, ML_defaultNormLog, MaxML,
|
||||
prevEntropy->matchlengthCTable,
|
||||
sizeof(prevEntropy->matchlengthCTable),
|
||||
entropyWorkspace, entropyWkspSize);
|
||||
if (ZSTD_isError(countSize)) {
|
||||
DEBUGLOG(3, "ZSTD_buildCTable for MatchLengths failed");
|
||||
stats.size = countSize;
|
||||
return stats;
|
||||
}
|
||||
if (stats.MLtype == set_compressed)
|
||||
stats.lastCountSize = countSize;
|
||||
op += countSize;
|
||||
assert(op <= oend);
|
||||
} }
|
||||
stats.size = (size_t)(op-ostart);
|
||||
return stats;
|
||||
}
|
||||
|
||||
/* ZSTD_entropyCompressSeqStore_internal():
|
||||
* compresses both literals and sequences
|
||||
* Returns compressed size of block, or a zstd error.
|
||||
*/
|
||||
#define SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO 20
|
||||
MEM_STATIC size_t
|
||||
ZSTD_entropyCompressSeqStore_internal(
|
||||
void* dst, size_t dstCapacity,
|
||||
@@ -2766,150 +2662,13 @@ ZSTD_entropyCompressSeqStore_internal(
|
||||
void* entropyWorkspace, size_t entropyWkspSize,
|
||||
const int bmi2)
|
||||
{
|
||||
ZSTD_strategy const strategy = cctxParams->cParams.strategy;
|
||||
unsigned* count = (unsigned*)entropyWorkspace;
|
||||
FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable;
|
||||
FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable;
|
||||
FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable;
|
||||
const SeqDef* const sequences = seqStorePtr->sequencesStart;
|
||||
const size_t nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
|
||||
const BYTE* const ofCodeTable = seqStorePtr->ofCode;
|
||||
const BYTE* const llCodeTable = seqStorePtr->llCode;
|
||||
const BYTE* const mlCodeTable = seqStorePtr->mlCode;
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* const oend = ostart + dstCapacity;
|
||||
BYTE* op = ostart;
|
||||
size_t lastCountSize;
|
||||
int longOffsets = 0;
|
||||
|
||||
entropyWorkspace = count + (MaxSeq + 1);
|
||||
entropyWkspSize -= (MaxSeq + 1) * sizeof(*count);
|
||||
|
||||
DEBUGLOG(5, "ZSTD_entropyCompressSeqStore_internal (nbSeq=%zu, dstCapacity=%zu)", nbSeq, dstCapacity);
|
||||
ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog)));
|
||||
assert(entropyWkspSize >= HUF_WORKSPACE_SIZE);
|
||||
|
||||
/* Compress literals */
|
||||
{ size_t const numSequences = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
|
||||
/* Base suspicion of uncompressibility on ratio of literals to sequences */
|
||||
int const suspectUncompressible = (numSequences == 0) || (litSize / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO);
|
||||
|
||||
size_t const cSize = ZSTD_compressLiterals(
|
||||
op, dstCapacity,
|
||||
literals, litSize,
|
||||
entropyWorkspace, entropyWkspSize,
|
||||
&prevEntropy->huf, &nextEntropy->huf,
|
||||
cctxParams->cParams.strategy,
|
||||
ZSTD_literalsCompressionIsDisabled(cctxParams),
|
||||
suspectUncompressible, bmi2);
|
||||
FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed");
|
||||
assert(cSize <= dstCapacity);
|
||||
op += cSize;
|
||||
}
|
||||
|
||||
/* Sequences Header */
|
||||
RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/,
|
||||
dstSize_tooSmall, "Can't fit seq hdr in output buf!");
|
||||
if (nbSeq < 128) {
|
||||
*op++ = (BYTE)nbSeq;
|
||||
} else if (nbSeq < LONGNBSEQ) {
|
||||
op[0] = (BYTE)((nbSeq>>8) + 0x80);
|
||||
op[1] = (BYTE)nbSeq;
|
||||
op+=2;
|
||||
} else {
|
||||
op[0]=0xFF;
|
||||
MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ));
|
||||
op+=3;
|
||||
}
|
||||
assert(op <= oend);
|
||||
if (nbSeq==0) {
|
||||
/* Copy the old tables over as if we repeated them */
|
||||
ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse));
|
||||
return (size_t)(op - ostart);
|
||||
}
|
||||
{ BYTE* const seqHead = op++;
|
||||
/* build stats for sequences */
|
||||
const ZSTD_symbolEncodingTypeStats_t stats =
|
||||
ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq,
|
||||
&prevEntropy->fse, &nextEntropy->fse,
|
||||
op, oend,
|
||||
strategy, count,
|
||||
entropyWorkspace, entropyWkspSize);
|
||||
FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!");
|
||||
*seqHead = (BYTE)((stats.LLtype<<6) + (stats.Offtype<<4) + (stats.MLtype<<2));
|
||||
lastCountSize = stats.lastCountSize;
|
||||
op += stats.size;
|
||||
longOffsets = stats.longOffsets;
|
||||
}
|
||||
|
||||
{ size_t const bitstreamSize = ZSTD_encodeSequences(
|
||||
op, (size_t)(oend - op),
|
||||
CTable_MatchLength, mlCodeTable,
|
||||
CTable_OffsetBits, ofCodeTable,
|
||||
CTable_LitLength, llCodeTable,
|
||||
sequences, nbSeq,
|
||||
longOffsets, bmi2);
|
||||
FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed");
|
||||
op += bitstreamSize;
|
||||
assert(op <= oend);
|
||||
/* zstd versions <= 1.3.4 mistakenly report corruption when
|
||||
* FSE_readNCount() receives a buffer < 4 bytes.
|
||||
* Fixed by https://github.com/facebook/zstd/pull/1146.
|
||||
* This can happen when the last set_compressed table present is 2
|
||||
* bytes and the bitstream is only one byte.
|
||||
* In this exceedingly rare case, we will simply emit an uncompressed
|
||||
* block, since it isn't worth optimizing.
|
||||
*/
|
||||
if (lastCountSize && (lastCountSize + bitstreamSize) < 4) {
|
||||
/* lastCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */
|
||||
assert(lastCountSize + bitstreamSize == 3);
|
||||
DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by "
|
||||
"emitting an uncompressed block.");
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
DEBUGLOG(5, "compressed block size : %u", (unsigned)(op - ostart));
|
||||
return (size_t)(op - ostart);
|
||||
}
|
||||
|
||||
static size_t
|
||||
ZSTD_entropyCompressSeqStore_wExtLitBuffer(
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* literals, size_t litSize,
|
||||
size_t blockSize,
|
||||
const SeqStore_t* seqStorePtr,
|
||||
const ZSTD_entropyCTables_t* prevEntropy,
|
||||
ZSTD_entropyCTables_t* nextEntropy,
|
||||
const ZSTD_CCtx_params* cctxParams,
|
||||
void* entropyWorkspace, size_t entropyWkspSize,
|
||||
int bmi2)
|
||||
{
|
||||
size_t const cSize = ZSTD_entropyCompressSeqStore_internal(
|
||||
dst, dstCapacity,
|
||||
literals, litSize,
|
||||
seqStorePtr, prevEntropy, nextEntropy, cctxParams,
|
||||
entropyWorkspace, entropyWkspSize, bmi2);
|
||||
if (cSize == 0) return 0;
|
||||
/* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block.
|
||||
* Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block.
|
||||
*/
|
||||
if ((cSize == ERROR(dstSize_tooSmall)) & (blockSize <= dstCapacity)) {
|
||||
DEBUGLOG(4, "not enough dstCapacity (%zu) for ZSTD_entropyCompressSeqStore_internal()=> do not compress block", dstCapacity);
|
||||
return 0; /* block not compressed */
|
||||
}
|
||||
FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSeqStore_internal failed");
|
||||
|
||||
/* Check compressibility */
|
||||
{ size_t const maxCSize = blockSize - ZSTD_minGain(blockSize, cctxParams->cParams.strategy);
|
||||
if (cSize >= maxCSize) return 0; /* block not compressed */
|
||||
}
|
||||
DEBUGLOG(5, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize);
|
||||
/* libzstd decoder before > v1.5.4 is not compatible with compressed blocks of size ZSTD_BLOCKSIZE_MAX exactly.
|
||||
* This restriction is indirectly already fulfilled by respecting ZSTD_minGain() condition above.
|
||||
*/
|
||||
assert(cSize < ZSTD_BLOCKSIZE_MAX);
|
||||
return cSize;
|
||||
return ZSTD_rust_entropyCompressSeqStore_internal(
|
||||
dst, dstCapacity,
|
||||
literals, litSize,
|
||||
seqStorePtr, prevEntropy, nextEntropy,
|
||||
(int)cctxParams->cParams.strategy,
|
||||
ZSTD_literalsCompressionIsDisabled(cctxParams),
|
||||
entropyWorkspace, entropyWkspSize, bmi2);
|
||||
}
|
||||
|
||||
static size_t
|
||||
@@ -2923,13 +2682,12 @@ ZSTD_entropyCompressSeqStore(
|
||||
void* entropyWorkspace, size_t entropyWkspSize,
|
||||
int bmi2)
|
||||
{
|
||||
return ZSTD_entropyCompressSeqStore_wExtLitBuffer(
|
||||
return ZSTD_rust_entropyCompressSeqStore(
|
||||
seqStorePtr, prevEntropy, nextEntropy,
|
||||
(int)cctxParams->cParams.strategy,
|
||||
ZSTD_literalsCompressionIsDisabled(cctxParams),
|
||||
dst, dstCapacity,
|
||||
seqStorePtr->litStart, (size_t)(seqStorePtr->lit - seqStorePtr->litStart),
|
||||
srcSize,
|
||||
seqStorePtr,
|
||||
prevEntropy, nextEntropy,
|
||||
cctxParams,
|
||||
entropyWorkspace, entropyWkspSize,
|
||||
bmi2);
|
||||
}
|
||||
@@ -3299,87 +3057,8 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
|
||||
|
||||
static size_t ZSTD_copyBlockSequences(SeqCollector* seqCollector, const SeqStore_t* seqStore, const U32 prevRepcodes[ZSTD_REP_NUM])
|
||||
{
|
||||
const SeqDef* inSeqs = seqStore->sequencesStart;
|
||||
const size_t nbInSequences = (size_t)(seqStore->sequences - inSeqs);
|
||||
const size_t nbInLiterals = (size_t)(seqStore->lit - seqStore->litStart);
|
||||
|
||||
ZSTD_Sequence* outSeqs = seqCollector->seqIndex == 0 ? seqCollector->seqStart : seqCollector->seqStart + seqCollector->seqIndex;
|
||||
const size_t nbOutSequences = nbInSequences + 1;
|
||||
size_t nbOutLiterals = 0;
|
||||
Repcodes_t repcodes;
|
||||
size_t i;
|
||||
|
||||
/* Bounds check that we have enough space for every input sequence
|
||||
* and the block delimiter
|
||||
*/
|
||||
assert(seqCollector->seqIndex <= seqCollector->maxSequences);
|
||||
RETURN_ERROR_IF(
|
||||
nbOutSequences > (size_t)(seqCollector->maxSequences - seqCollector->seqIndex),
|
||||
dstSize_tooSmall,
|
||||
"Not enough space to copy sequences");
|
||||
|
||||
ZSTD_memcpy(&repcodes, prevRepcodes, sizeof(repcodes));
|
||||
for (i = 0; i < nbInSequences; ++i) {
|
||||
U32 rawOffset;
|
||||
outSeqs[i].litLength = inSeqs[i].litLength;
|
||||
outSeqs[i].matchLength = inSeqs[i].mlBase + MINMATCH;
|
||||
outSeqs[i].rep = 0;
|
||||
|
||||
/* Handle the possible single length >= 64K
|
||||
* There can only be one because we add MINMATCH to every match length,
|
||||
* and blocks are at most 128K.
|
||||
*/
|
||||
if (i == seqStore->longLengthPos) {
|
||||
if (seqStore->longLengthType == ZSTD_llt_literalLength) {
|
||||
outSeqs[i].litLength += 0x10000;
|
||||
} else if (seqStore->longLengthType == ZSTD_llt_matchLength) {
|
||||
outSeqs[i].matchLength += 0x10000;
|
||||
}
|
||||
}
|
||||
|
||||
/* Determine the raw offset given the offBase, which may be a repcode. */
|
||||
if (OFFBASE_IS_REPCODE(inSeqs[i].offBase)) {
|
||||
const U32 repcode = OFFBASE_TO_REPCODE(inSeqs[i].offBase);
|
||||
assert(repcode > 0);
|
||||
outSeqs[i].rep = repcode;
|
||||
if (outSeqs[i].litLength != 0) {
|
||||
rawOffset = repcodes.rep[repcode - 1];
|
||||
} else {
|
||||
if (repcode == 3) {
|
||||
assert(repcodes.rep[0] > 1);
|
||||
rawOffset = repcodes.rep[0] - 1;
|
||||
} else {
|
||||
rawOffset = repcodes.rep[repcode];
|
||||
}
|
||||
}
|
||||
} else {
|
||||
rawOffset = OFFBASE_TO_OFFSET(inSeqs[i].offBase);
|
||||
}
|
||||
outSeqs[i].offset = rawOffset;
|
||||
|
||||
/* Update repcode history for the sequence */
|
||||
ZSTD_updateRep(repcodes.rep,
|
||||
inSeqs[i].offBase,
|
||||
inSeqs[i].litLength == 0);
|
||||
|
||||
nbOutLiterals += outSeqs[i].litLength;
|
||||
}
|
||||
/* Insert last literals (if any exist) in the block as a sequence with ml == off == 0.
|
||||
* If there are no last literals, then we'll emit (of: 0, ml: 0, ll: 0), which is a marker
|
||||
* for the block boundary, according to the API.
|
||||
*/
|
||||
assert(nbInLiterals >= nbOutLiterals);
|
||||
{
|
||||
const size_t lastLLSize = nbInLiterals - nbOutLiterals;
|
||||
outSeqs[nbInSequences].litLength = (U32)lastLLSize;
|
||||
outSeqs[nbInSequences].matchLength = 0;
|
||||
outSeqs[nbInSequences].offset = 0;
|
||||
assert(nbOutSequences == nbInSequences + 1);
|
||||
}
|
||||
seqCollector->seqIndex += nbOutSequences;
|
||||
assert(seqCollector->seqIndex <= seqCollector->maxSequences);
|
||||
|
||||
return 0;
|
||||
/* The implementation lives in rust/src/zstd_compress_stats.rs. */
|
||||
return ZSTD_rust_copyBlockSequences(seqCollector, seqStore, prevRepcodes);
|
||||
}
|
||||
|
||||
/* ZSTD_sequenceBound() lives in rust/src/zstd_compress_api.rs. */
|
||||
@@ -3475,178 +3154,14 @@ writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock)
|
||||
DEBUGLOG(5, "writeBlockHeader: cSize: %zu blockSize: %zu lastBlock: %u", cSize, blockSize, lastBlock);
|
||||
}
|
||||
|
||||
/** ZSTD_buildBlockEntropyStats_literals() :
|
||||
* Builds entropy for the literals.
|
||||
* Stores literals block type (raw, rle, compressed, repeat) and
|
||||
* huffman description table to hufMetadata.
|
||||
* Requires ENTROPY_WORKSPACE_SIZE workspace
|
||||
* @return : size of huffman description table, or an error code
|
||||
*/
|
||||
static size_t
|
||||
ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize,
|
||||
const ZSTD_hufCTables_t* prevHuf,
|
||||
ZSTD_hufCTables_t* nextHuf,
|
||||
ZSTD_hufCTablesMetadata_t* hufMetadata,
|
||||
const int literalsCompressionIsDisabled,
|
||||
void* workspace, size_t wkspSize,
|
||||
int hufFlags)
|
||||
{
|
||||
BYTE* const wkspStart = (BYTE*)workspace;
|
||||
BYTE* const wkspEnd = wkspStart + wkspSize;
|
||||
BYTE* const countWkspStart = wkspStart;
|
||||
unsigned* const countWksp = (unsigned*)workspace;
|
||||
const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned);
|
||||
BYTE* const nodeWksp = countWkspStart + countWkspSize;
|
||||
const size_t nodeWkspSize = (size_t)(wkspEnd - nodeWksp);
|
||||
unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX;
|
||||
unsigned huffLog = LitHufLog;
|
||||
HUF_repeat repeat = prevHuf->repeatMode;
|
||||
DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_literals (srcSize=%zu)", srcSize);
|
||||
|
||||
/* Prepare nextEntropy assuming reusing the existing table */
|
||||
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
||||
|
||||
if (literalsCompressionIsDisabled) {
|
||||
DEBUGLOG(5, "set_basic - disabled");
|
||||
hufMetadata->hType = set_basic;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* small ? don't even attempt compression (speed opt) */
|
||||
#ifndef COMPRESS_LITERALS_SIZE_MIN
|
||||
# define COMPRESS_LITERALS_SIZE_MIN 63 /* heuristic */
|
||||
#endif
|
||||
{ size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN;
|
||||
if (srcSize <= minLitSize) {
|
||||
DEBUGLOG(5, "set_basic - too small");
|
||||
hufMetadata->hType = set_basic;
|
||||
return 0;
|
||||
} }
|
||||
|
||||
/* Scan input and build symbol stats */
|
||||
{ size_t const largest =
|
||||
HIST_count_wksp (countWksp, &maxSymbolValue,
|
||||
(const BYTE*)src, srcSize,
|
||||
workspace, wkspSize);
|
||||
FORWARD_IF_ERROR(largest, "HIST_count_wksp failed");
|
||||
if (largest == srcSize) {
|
||||
/* only one literal symbol */
|
||||
DEBUGLOG(5, "set_rle");
|
||||
hufMetadata->hType = set_rle;
|
||||
return 0;
|
||||
}
|
||||
if (largest <= (srcSize >> 7)+4) {
|
||||
/* heuristic: likely not compressible */
|
||||
DEBUGLOG(5, "set_basic - no gain");
|
||||
hufMetadata->hType = set_basic;
|
||||
return 0;
|
||||
} }
|
||||
|
||||
/* Validate the previous Huffman table */
|
||||
if (repeat == HUF_repeat_check
|
||||
&& !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) {
|
||||
repeat = HUF_repeat_none;
|
||||
}
|
||||
|
||||
/* Build Huffman Tree */
|
||||
ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable));
|
||||
huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, nodeWksp, nodeWkspSize, nextHuf->CTable, countWksp, hufFlags);
|
||||
assert(huffLog <= LitHufLog);
|
||||
{ size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp,
|
||||
maxSymbolValue, huffLog,
|
||||
nodeWksp, nodeWkspSize);
|
||||
FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp");
|
||||
huffLog = (U32)maxBits;
|
||||
}
|
||||
{ /* Build and write the CTable */
|
||||
size_t const newCSize = HUF_estimateCompressedSize(
|
||||
(HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue);
|
||||
size_t const hSize = HUF_writeCTable_wksp(
|
||||
hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer),
|
||||
(HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog,
|
||||
nodeWksp, nodeWkspSize);
|
||||
/* Check against repeating the previous CTable */
|
||||
if (repeat != HUF_repeat_none) {
|
||||
size_t const oldCSize = HUF_estimateCompressedSize(
|
||||
(HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue);
|
||||
if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) {
|
||||
DEBUGLOG(5, "set_repeat - smaller");
|
||||
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
||||
hufMetadata->hType = set_repeat;
|
||||
return 0;
|
||||
} }
|
||||
if (newCSize + hSize >= srcSize) {
|
||||
DEBUGLOG(5, "set_basic - no gains");
|
||||
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
|
||||
hufMetadata->hType = set_basic;
|
||||
return 0;
|
||||
}
|
||||
DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize);
|
||||
hufMetadata->hType = set_compressed;
|
||||
nextHuf->repeatMode = HUF_repeat_check;
|
||||
return hSize;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* ZSTD_buildDummySequencesStatistics():
|
||||
* Returns a ZSTD_symbolEncodingTypeStats_t with all encoding types as set_basic,
|
||||
* and updates nextEntropy to the appropriate repeatMode.
|
||||
*/
|
||||
static ZSTD_symbolEncodingTypeStats_t
|
||||
ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy)
|
||||
{
|
||||
ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0, 0};
|
||||
nextEntropy->litlength_repeatMode = FSE_repeat_none;
|
||||
nextEntropy->offcode_repeatMode = FSE_repeat_none;
|
||||
nextEntropy->matchlength_repeatMode = FSE_repeat_none;
|
||||
return stats;
|
||||
}
|
||||
|
||||
/** ZSTD_buildBlockEntropyStats_sequences() :
|
||||
* Builds entropy for the sequences.
|
||||
* Stores symbol compression modes and fse table to fseMetadata.
|
||||
* Requires ENTROPY_WORKSPACE_SIZE wksp.
|
||||
* @return : size of fse tables or error code */
|
||||
static size_t
|
||||
ZSTD_buildBlockEntropyStats_sequences(
|
||||
const SeqStore_t* seqStorePtr,
|
||||
const ZSTD_fseCTables_t* prevEntropy,
|
||||
ZSTD_fseCTables_t* nextEntropy,
|
||||
const ZSTD_CCtx_params* cctxParams,
|
||||
ZSTD_fseCTablesMetadata_t* fseMetadata,
|
||||
void* workspace, size_t wkspSize)
|
||||
{
|
||||
ZSTD_strategy const strategy = cctxParams->cParams.strategy;
|
||||
size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
|
||||
BYTE* const ostart = fseMetadata->fseTablesBuffer;
|
||||
BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer);
|
||||
BYTE* op = ostart;
|
||||
unsigned* countWorkspace = (unsigned*)workspace;
|
||||
unsigned* entropyWorkspace = countWorkspace + (MaxSeq + 1);
|
||||
size_t entropyWorkspaceSize = wkspSize - (MaxSeq + 1) * sizeof(*countWorkspace);
|
||||
ZSTD_symbolEncodingTypeStats_t stats;
|
||||
|
||||
DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_sequences (nbSeq=%zu)", nbSeq);
|
||||
stats = nbSeq != 0 ? ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq,
|
||||
prevEntropy, nextEntropy, op, oend,
|
||||
strategy, countWorkspace,
|
||||
entropyWorkspace, entropyWorkspaceSize)
|
||||
: ZSTD_buildDummySequencesStatistics(nextEntropy);
|
||||
FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!");
|
||||
fseMetadata->llType = (SymbolEncodingType_e) stats.LLtype;
|
||||
fseMetadata->ofType = (SymbolEncodingType_e) stats.Offtype;
|
||||
fseMetadata->mlType = (SymbolEncodingType_e) stats.MLtype;
|
||||
fseMetadata->lastCountSize = stats.lastCountSize;
|
||||
return stats.size;
|
||||
}
|
||||
|
||||
|
||||
/** ZSTD_buildBlockEntropyStats() :
|
||||
* Builds entropy for the block.
|
||||
* Requires workspace size ENTROPY_WORKSPACE_SIZE
|
||||
* @return : 0 on success, or an error code
|
||||
* Note : also employed in superblock
|
||||
*
|
||||
* The implementation, together with its literals/sequences/dummy helpers,
|
||||
* lives in rust/src/zstd_compress_stats.rs.
|
||||
*/
|
||||
size_t ZSTD_buildBlockEntropyStats(
|
||||
const SeqStore_t* seqStorePtr,
|
||||
@@ -3656,26 +3171,12 @@ size_t ZSTD_buildBlockEntropyStats(
|
||||
ZSTD_entropyCTablesMetadata_t* entropyMetadata,
|
||||
void* workspace, size_t wkspSize)
|
||||
{
|
||||
size_t const litSize = (size_t)(seqStorePtr->lit - seqStorePtr->litStart);
|
||||
int const huf_useOptDepth = (cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD);
|
||||
int const hufFlags = huf_useOptDepth ? HUF_flags_optimalDepth : 0;
|
||||
|
||||
entropyMetadata->hufMetadata.hufDesSize =
|
||||
ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize,
|
||||
&prevEntropy->huf, &nextEntropy->huf,
|
||||
&entropyMetadata->hufMetadata,
|
||||
ZSTD_literalsCompressionIsDisabled(cctxParams),
|
||||
workspace, wkspSize, hufFlags);
|
||||
|
||||
FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed");
|
||||
entropyMetadata->fseMetadata.fseTablesSize =
|
||||
ZSTD_buildBlockEntropyStats_sequences(seqStorePtr,
|
||||
&prevEntropy->fse, &nextEntropy->fse,
|
||||
cctxParams,
|
||||
&entropyMetadata->fseMetadata,
|
||||
workspace, wkspSize);
|
||||
FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildBlockEntropyStats_sequences failed");
|
||||
return 0;
|
||||
return ZSTD_rust_buildBlockEntropyStats(
|
||||
seqStorePtr, prevEntropy, nextEntropy,
|
||||
(int)cctxParams->cParams.strategy,
|
||||
ZSTD_literalsCompressionIsDisabled(cctxParams),
|
||||
entropyMetadata,
|
||||
workspace, wkspSize);
|
||||
}
|
||||
|
||||
/* Returns the size estimate for the literals section (header + content) of a block */
|
||||
|
||||
@@ -29,7 +29,7 @@ size_t ZSTD_rust_compressSuperBlock(
|
||||
const SeqStore_t* seqStore,
|
||||
const ZSTD_compressedBlockState_t* prevCBlock,
|
||||
ZSTD_compressedBlockState_t* nextCBlock,
|
||||
const ZSTD_CCtx_params* cctxParams,
|
||||
int strategy, int disableLiteralCompression,
|
||||
void* workspace, size_t wkspSize,
|
||||
int bmi2, U32 windowLog, size_t targetCBlockSize,
|
||||
void* dst, size_t dstCapacity,
|
||||
@@ -45,7 +45,8 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
|
||||
&zc->seqStore,
|
||||
zc->blockState.prevCBlock,
|
||||
zc->blockState.nextCBlock,
|
||||
&zc->appliedParams,
|
||||
(int)zc->appliedParams.cParams.strategy,
|
||||
ZSTD_literalsCompressionIsDisabled(&zc->appliedParams),
|
||||
zc->tmpWorkspace, zc->tmpWkspSize,
|
||||
zc->bmi2, zc->appliedParams.cParams.windowLog,
|
||||
zc->appliedParams.targetCBlockSize,
|
||||
|
||||
+1886
-5
@@ -24,9 +24,1890 @@
|
||||
* OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
/* divsufsort() is implemented in rust/src/divsufsort.rs, which provides the
|
||||
* symbol directly. This translation unit keeps the header's prototypes in
|
||||
* the build so the dictionary builder continues to compile against the
|
||||
* original interface. divbwt() has no callers in zstd and is declaration-
|
||||
* only; it moves to Rust if a user ever appears. */
|
||||
/*- Compiler specifics -*/
|
||||
#ifdef __clang__
|
||||
#pragma clang diagnostic ignored "-Wshorten-64-to-32"
|
||||
#endif
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
# pragma warning(disable : 4244)
|
||||
# pragma warning(disable : 4127) /* C4127 : Condition expression is constant */
|
||||
#endif
|
||||
|
||||
|
||||
/*- Dependencies -*/
|
||||
#include <assert.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "divsufsort.h"
|
||||
|
||||
/*- Constants -*/
|
||||
#if defined(INLINE)
|
||||
# undef INLINE
|
||||
#endif
|
||||
#if !defined(INLINE)
|
||||
# define INLINE __inline
|
||||
#endif
|
||||
#if defined(ALPHABET_SIZE) && (ALPHABET_SIZE < 1)
|
||||
# undef ALPHABET_SIZE
|
||||
#endif
|
||||
#if !defined(ALPHABET_SIZE)
|
||||
# define ALPHABET_SIZE (256)
|
||||
#endif
|
||||
#define BUCKET_A_SIZE (ALPHABET_SIZE)
|
||||
#define BUCKET_B_SIZE (ALPHABET_SIZE * ALPHABET_SIZE)
|
||||
#if defined(SS_INSERTIONSORT_THRESHOLD)
|
||||
# if SS_INSERTIONSORT_THRESHOLD < 1
|
||||
# undef SS_INSERTIONSORT_THRESHOLD
|
||||
# define SS_INSERTIONSORT_THRESHOLD (1)
|
||||
# endif
|
||||
#else
|
||||
# define SS_INSERTIONSORT_THRESHOLD (8)
|
||||
#endif
|
||||
#if defined(SS_BLOCKSIZE)
|
||||
# if SS_BLOCKSIZE < 0
|
||||
# undef SS_BLOCKSIZE
|
||||
# define SS_BLOCKSIZE (0)
|
||||
# elif 32768 <= SS_BLOCKSIZE
|
||||
# undef SS_BLOCKSIZE
|
||||
# define SS_BLOCKSIZE (32767)
|
||||
# endif
|
||||
#else
|
||||
# define SS_BLOCKSIZE (1024)
|
||||
#endif
|
||||
/* minstacksize = log(SS_BLOCKSIZE) / log(3) * 2 */
|
||||
#if SS_BLOCKSIZE == 0
|
||||
# define SS_MISORT_STACKSIZE (96)
|
||||
#elif SS_BLOCKSIZE <= 4096
|
||||
# define SS_MISORT_STACKSIZE (16)
|
||||
#else
|
||||
# define SS_MISORT_STACKSIZE (24)
|
||||
#endif
|
||||
#define SS_SMERGE_STACKSIZE (32)
|
||||
#define TR_INSERTIONSORT_THRESHOLD (8)
|
||||
#define TR_STACKSIZE (64)
|
||||
|
||||
|
||||
/*- Macros -*/
|
||||
#ifndef SWAP
|
||||
# define SWAP(_a, _b) do { t = (_a); (_a) = (_b); (_b) = t; } while(0)
|
||||
#endif /* SWAP */
|
||||
#ifndef MIN
|
||||
# define MIN(_a, _b) (((_a) < (_b)) ? (_a) : (_b))
|
||||
#endif /* MIN */
|
||||
#ifndef MAX
|
||||
# define MAX(_a, _b) (((_a) > (_b)) ? (_a) : (_b))
|
||||
#endif /* MAX */
|
||||
#define STACK_PUSH(_a, _b, _c, _d)\
|
||||
do {\
|
||||
assert(ssize < STACK_SIZE);\
|
||||
stack[ssize].a = (_a), stack[ssize].b = (_b),\
|
||||
stack[ssize].c = (_c), stack[ssize++].d = (_d);\
|
||||
} while(0)
|
||||
#define STACK_PUSH5(_a, _b, _c, _d, _e)\
|
||||
do {\
|
||||
assert(ssize < STACK_SIZE);\
|
||||
stack[ssize].a = (_a), stack[ssize].b = (_b),\
|
||||
stack[ssize].c = (_c), stack[ssize].d = (_d), stack[ssize++].e = (_e);\
|
||||
} while(0)
|
||||
#define STACK_POP(_a, _b, _c, _d)\
|
||||
do {\
|
||||
assert(0 <= ssize);\
|
||||
if(ssize == 0) { return; }\
|
||||
(_a) = stack[--ssize].a, (_b) = stack[ssize].b,\
|
||||
(_c) = stack[ssize].c, (_d) = stack[ssize].d;\
|
||||
} while(0)
|
||||
#define STACK_POP5(_a, _b, _c, _d, _e)\
|
||||
do {\
|
||||
assert(0 <= ssize);\
|
||||
if(ssize == 0) { return; }\
|
||||
(_a) = stack[--ssize].a, (_b) = stack[ssize].b,\
|
||||
(_c) = stack[ssize].c, (_d) = stack[ssize].d, (_e) = stack[ssize].e;\
|
||||
} while(0)
|
||||
#define BUCKET_A(_c0) bucket_A[(_c0)]
|
||||
#if ALPHABET_SIZE == 256
|
||||
#define BUCKET_B(_c0, _c1) (bucket_B[((_c1) << 8) | (_c0)])
|
||||
#define BUCKET_BSTAR(_c0, _c1) (bucket_B[((_c0) << 8) | (_c1)])
|
||||
#else
|
||||
#define BUCKET_B(_c0, _c1) (bucket_B[(_c1) * ALPHABET_SIZE + (_c0)])
|
||||
#define BUCKET_BSTAR(_c0, _c1) (bucket_B[(_c0) * ALPHABET_SIZE + (_c1)])
|
||||
#endif
|
||||
|
||||
|
||||
/*- Private Functions -*/
|
||||
|
||||
static const int lg_table[256]= {
|
||||
-1,0,1,1,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,
|
||||
5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,
|
||||
6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
|
||||
6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
|
||||
7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
|
||||
7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
|
||||
7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
|
||||
7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7
|
||||
};
|
||||
|
||||
#if (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE)
|
||||
|
||||
static INLINE
|
||||
int
|
||||
ss_ilg(int n) {
|
||||
#if SS_BLOCKSIZE == 0
|
||||
return (n & 0xffff0000) ?
|
||||
((n & 0xff000000) ?
|
||||
24 + lg_table[(n >> 24) & 0xff] :
|
||||
16 + lg_table[(n >> 16) & 0xff]) :
|
||||
((n & 0x0000ff00) ?
|
||||
8 + lg_table[(n >> 8) & 0xff] :
|
||||
0 + lg_table[(n >> 0) & 0xff]);
|
||||
#elif SS_BLOCKSIZE < 256
|
||||
return lg_table[n];
|
||||
#else
|
||||
return (n & 0xff00) ?
|
||||
8 + lg_table[(n >> 8) & 0xff] :
|
||||
0 + lg_table[(n >> 0) & 0xff];
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif /* (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) */
|
||||
|
||||
#if SS_BLOCKSIZE != 0
|
||||
|
||||
static const int sqq_table[256] = {
|
||||
0, 16, 22, 27, 32, 35, 39, 42, 45, 48, 50, 53, 55, 57, 59, 61,
|
||||
64, 65, 67, 69, 71, 73, 75, 76, 78, 80, 81, 83, 84, 86, 87, 89,
|
||||
90, 91, 93, 94, 96, 97, 98, 99, 101, 102, 103, 104, 106, 107, 108, 109,
|
||||
110, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126,
|
||||
128, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142,
|
||||
143, 144, 144, 145, 146, 147, 148, 149, 150, 150, 151, 152, 153, 154, 155, 155,
|
||||
156, 157, 158, 159, 160, 160, 161, 162, 163, 163, 164, 165, 166, 167, 167, 168,
|
||||
169, 170, 170, 171, 172, 173, 173, 174, 175, 176, 176, 177, 178, 178, 179, 180,
|
||||
181, 181, 182, 183, 183, 184, 185, 185, 186, 187, 187, 188, 189, 189, 190, 191,
|
||||
192, 192, 193, 193, 194, 195, 195, 196, 197, 197, 198, 199, 199, 200, 201, 201,
|
||||
202, 203, 203, 204, 204, 205, 206, 206, 207, 208, 208, 209, 209, 210, 211, 211,
|
||||
212, 212, 213, 214, 214, 215, 215, 216, 217, 217, 218, 218, 219, 219, 220, 221,
|
||||
221, 222, 222, 223, 224, 224, 225, 225, 226, 226, 227, 227, 228, 229, 229, 230,
|
||||
230, 231, 231, 232, 232, 233, 234, 234, 235, 235, 236, 236, 237, 237, 238, 238,
|
||||
239, 240, 240, 241, 241, 242, 242, 243, 243, 244, 244, 245, 245, 246, 246, 247,
|
||||
247, 248, 248, 249, 249, 250, 250, 251, 251, 252, 252, 253, 253, 254, 254, 255
|
||||
};
|
||||
|
||||
static INLINE
|
||||
int
|
||||
ss_isqrt(int x) {
|
||||
int y, e;
|
||||
|
||||
if(x >= (SS_BLOCKSIZE * SS_BLOCKSIZE)) { return SS_BLOCKSIZE; }
|
||||
e = (x & 0xffff0000) ?
|
||||
((x & 0xff000000) ?
|
||||
24 + lg_table[(x >> 24) & 0xff] :
|
||||
16 + lg_table[(x >> 16) & 0xff]) :
|
||||
((x & 0x0000ff00) ?
|
||||
8 + lg_table[(x >> 8) & 0xff] :
|
||||
0 + lg_table[(x >> 0) & 0xff]);
|
||||
|
||||
if(e >= 16) {
|
||||
y = sqq_table[x >> ((e - 6) - (e & 1))] << ((e >> 1) - 7);
|
||||
if(e >= 24) { y = (y + 1 + x / y) >> 1; }
|
||||
y = (y + 1 + x / y) >> 1;
|
||||
} else if(e >= 8) {
|
||||
y = (sqq_table[x >> ((e - 6) - (e & 1))] >> (7 - (e >> 1))) + 1;
|
||||
} else {
|
||||
return sqq_table[x] >> 4;
|
||||
}
|
||||
|
||||
return (x < (y * y)) ? y - 1 : y;
|
||||
}
|
||||
|
||||
#endif /* SS_BLOCKSIZE != 0 */
|
||||
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
/* Compares two suffixes. */
|
||||
static INLINE
|
||||
int
|
||||
ss_compare(const unsigned char *T,
|
||||
const int *p1, const int *p2,
|
||||
int depth) {
|
||||
const unsigned char *U1, *U2, *U1n, *U2n;
|
||||
|
||||
for(U1 = T + depth + *p1,
|
||||
U2 = T + depth + *p2,
|
||||
U1n = T + *(p1 + 1) + 2,
|
||||
U2n = T + *(p2 + 1) + 2;
|
||||
(U1 < U1n) && (U2 < U2n) && (*U1 == *U2);
|
||||
++U1, ++U2) {
|
||||
}
|
||||
|
||||
return U1 < U1n ?
|
||||
(U2 < U2n ? *U1 - *U2 : 1) :
|
||||
(U2 < U2n ? -1 : 0);
|
||||
}
|
||||
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
#if (SS_BLOCKSIZE != 1) && (SS_INSERTIONSORT_THRESHOLD != 1)
|
||||
|
||||
/* Insertionsort for small size groups */
|
||||
static
|
||||
void
|
||||
ss_insertionsort(const unsigned char *T, const int *PA,
|
||||
int *first, int *last, int depth) {
|
||||
int *i, *j;
|
||||
int t;
|
||||
int r;
|
||||
|
||||
for(i = last - 2; first <= i; --i) {
|
||||
for(t = *i, j = i + 1; 0 < (r = ss_compare(T, PA + t, PA + *j, depth));) {
|
||||
do { *(j - 1) = *j; } while((++j < last) && (*j < 0));
|
||||
if(last <= j) { break; }
|
||||
}
|
||||
if(r == 0) { *j = ~*j; }
|
||||
*(j - 1) = t;
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* (SS_BLOCKSIZE != 1) && (SS_INSERTIONSORT_THRESHOLD != 1) */
|
||||
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
#if (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE)
|
||||
|
||||
static INLINE
|
||||
void
|
||||
ss_fixdown(const unsigned char *Td, const int *PA,
|
||||
int *SA, int i, int size) {
|
||||
int j, k;
|
||||
int v;
|
||||
int c, d, e;
|
||||
|
||||
for(v = SA[i], c = Td[PA[v]]; (j = 2 * i + 1) < size; SA[i] = SA[k], i = k) {
|
||||
d = Td[PA[SA[k = j++]]];
|
||||
if(d < (e = Td[PA[SA[j]]])) { k = j; d = e; }
|
||||
if(d <= c) { break; }
|
||||
}
|
||||
SA[i] = v;
|
||||
}
|
||||
|
||||
/* Simple top-down heapsort. */
|
||||
static
|
||||
void
|
||||
ss_heapsort(const unsigned char *Td, const int *PA, int *SA, int size) {
|
||||
int i, m;
|
||||
int t;
|
||||
|
||||
m = size;
|
||||
if((size % 2) == 0) {
|
||||
m--;
|
||||
if(Td[PA[SA[m / 2]]] < Td[PA[SA[m]]]) { SWAP(SA[m], SA[m / 2]); }
|
||||
}
|
||||
|
||||
for(i = m / 2 - 1; 0 <= i; --i) { ss_fixdown(Td, PA, SA, i, m); }
|
||||
if((size % 2) == 0) { SWAP(SA[0], SA[m]); ss_fixdown(Td, PA, SA, 0, m); }
|
||||
for(i = m - 1; 0 < i; --i) {
|
||||
t = SA[0], SA[0] = SA[i];
|
||||
ss_fixdown(Td, PA, SA, 0, i);
|
||||
SA[i] = t;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
/* Returns the median of three elements. */
|
||||
static INLINE
|
||||
int *
|
||||
ss_median3(const unsigned char *Td, const int *PA,
|
||||
int *v1, int *v2, int *v3) {
|
||||
int *t;
|
||||
if(Td[PA[*v1]] > Td[PA[*v2]]) { SWAP(v1, v2); }
|
||||
if(Td[PA[*v2]] > Td[PA[*v3]]) {
|
||||
if(Td[PA[*v1]] > Td[PA[*v3]]) { return v1; }
|
||||
else { return v3; }
|
||||
}
|
||||
return v2;
|
||||
}
|
||||
|
||||
/* Returns the median of five elements. */
|
||||
static INLINE
|
||||
int *
|
||||
ss_median5(const unsigned char *Td, const int *PA,
|
||||
int *v1, int *v2, int *v3, int *v4, int *v5) {
|
||||
int *t;
|
||||
if(Td[PA[*v2]] > Td[PA[*v3]]) { SWAP(v2, v3); }
|
||||
if(Td[PA[*v4]] > Td[PA[*v5]]) { SWAP(v4, v5); }
|
||||
if(Td[PA[*v2]] > Td[PA[*v4]]) { SWAP(v2, v4); SWAP(v3, v5); }
|
||||
if(Td[PA[*v1]] > Td[PA[*v3]]) { SWAP(v1, v3); }
|
||||
if(Td[PA[*v1]] > Td[PA[*v4]]) { SWAP(v1, v4); SWAP(v3, v5); }
|
||||
if(Td[PA[*v3]] > Td[PA[*v4]]) { return v4; }
|
||||
return v3;
|
||||
}
|
||||
|
||||
/* Returns the pivot element. */
|
||||
static INLINE
|
||||
int *
|
||||
ss_pivot(const unsigned char *Td, const int *PA, int *first, int *last) {
|
||||
int *middle;
|
||||
int t;
|
||||
|
||||
t = last - first;
|
||||
middle = first + t / 2;
|
||||
|
||||
if(t <= 512) {
|
||||
if(t <= 32) {
|
||||
return ss_median3(Td, PA, first, middle, last - 1);
|
||||
} else {
|
||||
t >>= 2;
|
||||
return ss_median5(Td, PA, first, first + t, middle, last - 1 - t, last - 1);
|
||||
}
|
||||
}
|
||||
t >>= 3;
|
||||
first = ss_median3(Td, PA, first, first + t, first + (t << 1));
|
||||
middle = ss_median3(Td, PA, middle - t, middle, middle + t);
|
||||
last = ss_median3(Td, PA, last - 1 - (t << 1), last - 1 - t, last - 1);
|
||||
return ss_median3(Td, PA, first, middle, last);
|
||||
}
|
||||
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
/* Binary partition for substrings. */
|
||||
static INLINE
|
||||
int *
|
||||
ss_partition(const int *PA,
|
||||
int *first, int *last, int depth) {
|
||||
int *a, *b;
|
||||
int t;
|
||||
for(a = first - 1, b = last;;) {
|
||||
for(; (++a < b) && ((PA[*a] + depth) >= (PA[*a + 1] + 1));) { *a = ~*a; }
|
||||
for(; (a < --b) && ((PA[*b] + depth) < (PA[*b + 1] + 1));) { }
|
||||
if(b <= a) { break; }
|
||||
t = ~*b;
|
||||
*b = *a;
|
||||
*a = t;
|
||||
}
|
||||
if(first < a) { *first = ~*first; }
|
||||
return a;
|
||||
}
|
||||
|
||||
/* Multikey introsort for medium size groups. */
|
||||
static
|
||||
void
|
||||
ss_mintrosort(const unsigned char *T, const int *PA,
|
||||
int *first, int *last,
|
||||
int depth) {
|
||||
#define STACK_SIZE SS_MISORT_STACKSIZE
|
||||
struct { int *a, *b, c; int d; } stack[STACK_SIZE];
|
||||
const unsigned char *Td;
|
||||
int *a, *b, *c, *d, *e, *f;
|
||||
int s, t;
|
||||
int ssize;
|
||||
int limit;
|
||||
int v, x = 0;
|
||||
|
||||
for(ssize = 0, limit = ss_ilg(last - first);;) {
|
||||
|
||||
if((last - first) <= SS_INSERTIONSORT_THRESHOLD) {
|
||||
#if 1 < SS_INSERTIONSORT_THRESHOLD
|
||||
if(1 < (last - first)) { ss_insertionsort(T, PA, first, last, depth); }
|
||||
#endif
|
||||
STACK_POP(first, last, depth, limit);
|
||||
continue;
|
||||
}
|
||||
|
||||
Td = T + depth;
|
||||
if(limit-- == 0) { ss_heapsort(Td, PA, first, last - first); }
|
||||
if(limit < 0) {
|
||||
for(a = first + 1, v = Td[PA[*first]]; a < last; ++a) {
|
||||
if((x = Td[PA[*a]]) != v) {
|
||||
if(1 < (a - first)) { break; }
|
||||
v = x;
|
||||
first = a;
|
||||
}
|
||||
}
|
||||
if(Td[PA[*first] - 1] < v) {
|
||||
first = ss_partition(PA, first, a, depth);
|
||||
}
|
||||
if((a - first) <= (last - a)) {
|
||||
if(1 < (a - first)) {
|
||||
STACK_PUSH(a, last, depth, -1);
|
||||
last = a, depth += 1, limit = ss_ilg(a - first);
|
||||
} else {
|
||||
first = a, limit = -1;
|
||||
}
|
||||
} else {
|
||||
if(1 < (last - a)) {
|
||||
STACK_PUSH(first, a, depth + 1, ss_ilg(a - first));
|
||||
first = a, limit = -1;
|
||||
} else {
|
||||
last = a, depth += 1, limit = ss_ilg(a - first);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
/* choose pivot */
|
||||
a = ss_pivot(Td, PA, first, last);
|
||||
v = Td[PA[*a]];
|
||||
SWAP(*first, *a);
|
||||
|
||||
/* partition */
|
||||
for(b = first; (++b < last) && ((x = Td[PA[*b]]) == v);) { }
|
||||
if(((a = b) < last) && (x < v)) {
|
||||
for(; (++b < last) && ((x = Td[PA[*b]]) <= v);) {
|
||||
if(x == v) { SWAP(*b, *a); ++a; }
|
||||
}
|
||||
}
|
||||
for(c = last; (b < --c) && ((x = Td[PA[*c]]) == v);) { }
|
||||
if((b < (d = c)) && (x > v)) {
|
||||
for(; (b < --c) && ((x = Td[PA[*c]]) >= v);) {
|
||||
if(x == v) { SWAP(*c, *d); --d; }
|
||||
}
|
||||
}
|
||||
for(; b < c;) {
|
||||
SWAP(*b, *c);
|
||||
for(; (++b < c) && ((x = Td[PA[*b]]) <= v);) {
|
||||
if(x == v) { SWAP(*b, *a); ++a; }
|
||||
}
|
||||
for(; (b < --c) && ((x = Td[PA[*c]]) >= v);) {
|
||||
if(x == v) { SWAP(*c, *d); --d; }
|
||||
}
|
||||
}
|
||||
|
||||
if(a <= d) {
|
||||
c = b - 1;
|
||||
|
||||
if((s = a - first) > (t = b - a)) { s = t; }
|
||||
for(e = first, f = b - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); }
|
||||
if((s = d - c) > (t = last - d - 1)) { s = t; }
|
||||
for(e = b, f = last - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); }
|
||||
|
||||
a = first + (b - a), c = last - (d - c);
|
||||
b = (v <= Td[PA[*a] - 1]) ? a : ss_partition(PA, a, c, depth);
|
||||
|
||||
if((a - first) <= (last - c)) {
|
||||
if((last - c) <= (c - b)) {
|
||||
STACK_PUSH(b, c, depth + 1, ss_ilg(c - b));
|
||||
STACK_PUSH(c, last, depth, limit);
|
||||
last = a;
|
||||
} else if((a - first) <= (c - b)) {
|
||||
STACK_PUSH(c, last, depth, limit);
|
||||
STACK_PUSH(b, c, depth + 1, ss_ilg(c - b));
|
||||
last = a;
|
||||
} else {
|
||||
STACK_PUSH(c, last, depth, limit);
|
||||
STACK_PUSH(first, a, depth, limit);
|
||||
first = b, last = c, depth += 1, limit = ss_ilg(c - b);
|
||||
}
|
||||
} else {
|
||||
if((a - first) <= (c - b)) {
|
||||
STACK_PUSH(b, c, depth + 1, ss_ilg(c - b));
|
||||
STACK_PUSH(first, a, depth, limit);
|
||||
first = c;
|
||||
} else if((last - c) <= (c - b)) {
|
||||
STACK_PUSH(first, a, depth, limit);
|
||||
STACK_PUSH(b, c, depth + 1, ss_ilg(c - b));
|
||||
first = c;
|
||||
} else {
|
||||
STACK_PUSH(first, a, depth, limit);
|
||||
STACK_PUSH(c, last, depth, limit);
|
||||
first = b, last = c, depth += 1, limit = ss_ilg(c - b);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
limit += 1;
|
||||
if(Td[PA[*first] - 1] < v) {
|
||||
first = ss_partition(PA, first, last, depth);
|
||||
limit = ss_ilg(last - first);
|
||||
}
|
||||
depth += 1;
|
||||
}
|
||||
}
|
||||
#undef STACK_SIZE
|
||||
}
|
||||
|
||||
#endif /* (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) */
|
||||
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
#if SS_BLOCKSIZE != 0
|
||||
|
||||
static INLINE
|
||||
void
|
||||
ss_blockswap(int *a, int *b, int n) {
|
||||
int t;
|
||||
for(; 0 < n; --n, ++a, ++b) {
|
||||
t = *a, *a = *b, *b = t;
|
||||
}
|
||||
}
|
||||
|
||||
static INLINE
|
||||
void
|
||||
ss_rotate(int *first, int *middle, int *last) {
|
||||
int *a, *b, t;
|
||||
int l, r;
|
||||
l = middle - first, r = last - middle;
|
||||
for(; (0 < l) && (0 < r);) {
|
||||
if(l == r) { ss_blockswap(first, middle, l); break; }
|
||||
if(l < r) {
|
||||
a = last - 1, b = middle - 1;
|
||||
t = *a;
|
||||
do {
|
||||
*a-- = *b, *b-- = *a;
|
||||
if(b < first) {
|
||||
*a = t;
|
||||
last = a;
|
||||
if((r -= l + 1) <= l) { break; }
|
||||
a -= 1, b = middle - 1;
|
||||
t = *a;
|
||||
}
|
||||
} while(1);
|
||||
} else {
|
||||
a = first, b = middle;
|
||||
t = *a;
|
||||
do {
|
||||
*a++ = *b, *b++ = *a;
|
||||
if(last <= b) {
|
||||
*a = t;
|
||||
first = a + 1;
|
||||
if((l -= r + 1) <= r) { break; }
|
||||
a += 1, b = middle;
|
||||
t = *a;
|
||||
}
|
||||
} while(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
static
|
||||
void
|
||||
ss_inplacemerge(const unsigned char *T, const int *PA,
|
||||
int *first, int *middle, int *last,
|
||||
int depth) {
|
||||
const int *p;
|
||||
int *a, *b;
|
||||
int len, half;
|
||||
int q, r;
|
||||
int x;
|
||||
|
||||
for(;;) {
|
||||
if(*(last - 1) < 0) { x = 1; p = PA + ~*(last - 1); }
|
||||
else { x = 0; p = PA + *(last - 1); }
|
||||
for(a = first, len = middle - first, half = len >> 1, r = -1;
|
||||
0 < len;
|
||||
len = half, half >>= 1) {
|
||||
b = a + half;
|
||||
q = ss_compare(T, PA + ((0 <= *b) ? *b : ~*b), p, depth);
|
||||
if(q < 0) {
|
||||
a = b + 1;
|
||||
half -= (len & 1) ^ 1;
|
||||
} else {
|
||||
r = q;
|
||||
}
|
||||
}
|
||||
if(a < middle) {
|
||||
if(r == 0) { *a = ~*a; }
|
||||
ss_rotate(a, middle, last);
|
||||
last -= middle - a;
|
||||
middle = a;
|
||||
if(first == middle) { break; }
|
||||
}
|
||||
--last;
|
||||
if(x != 0) { while(*--last < 0) { } }
|
||||
if(middle == last) { break; }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
/* Merge-forward with internal buffer. */
|
||||
static
|
||||
void
|
||||
ss_mergeforward(const unsigned char *T, const int *PA,
|
||||
int *first, int *middle, int *last,
|
||||
int *buf, int depth) {
|
||||
int *a, *b, *c, *bufend;
|
||||
int t;
|
||||
int r;
|
||||
|
||||
bufend = buf + (middle - first) - 1;
|
||||
ss_blockswap(buf, first, middle - first);
|
||||
|
||||
for(t = *(a = first), b = buf, c = middle;;) {
|
||||
r = ss_compare(T, PA + *b, PA + *c, depth);
|
||||
if(r < 0) {
|
||||
do {
|
||||
*a++ = *b;
|
||||
if(bufend <= b) { *bufend = t; return; }
|
||||
*b++ = *a;
|
||||
} while(*b < 0);
|
||||
} else if(r > 0) {
|
||||
do {
|
||||
*a++ = *c, *c++ = *a;
|
||||
if(last <= c) {
|
||||
while(b < bufend) { *a++ = *b, *b++ = *a; }
|
||||
*a = *b, *b = t;
|
||||
return;
|
||||
}
|
||||
} while(*c < 0);
|
||||
} else {
|
||||
*c = ~*c;
|
||||
do {
|
||||
*a++ = *b;
|
||||
if(bufend <= b) { *bufend = t; return; }
|
||||
*b++ = *a;
|
||||
} while(*b < 0);
|
||||
|
||||
do {
|
||||
*a++ = *c, *c++ = *a;
|
||||
if(last <= c) {
|
||||
while(b < bufend) { *a++ = *b, *b++ = *a; }
|
||||
*a = *b, *b = t;
|
||||
return;
|
||||
}
|
||||
} while(*c < 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Merge-backward with internal buffer. */
|
||||
static
|
||||
void
|
||||
ss_mergebackward(const unsigned char *T, const int *PA,
|
||||
int *first, int *middle, int *last,
|
||||
int *buf, int depth) {
|
||||
const int *p1, *p2;
|
||||
int *a, *b, *c, *bufend;
|
||||
int t;
|
||||
int r;
|
||||
int x;
|
||||
|
||||
bufend = buf + (last - middle) - 1;
|
||||
ss_blockswap(buf, middle, last - middle);
|
||||
|
||||
x = 0;
|
||||
if(*bufend < 0) { p1 = PA + ~*bufend; x |= 1; }
|
||||
else { p1 = PA + *bufend; }
|
||||
if(*(middle - 1) < 0) { p2 = PA + ~*(middle - 1); x |= 2; }
|
||||
else { p2 = PA + *(middle - 1); }
|
||||
for(t = *(a = last - 1), b = bufend, c = middle - 1;;) {
|
||||
r = ss_compare(T, p1, p2, depth);
|
||||
if(0 < r) {
|
||||
if(x & 1) { do { *a-- = *b, *b-- = *a; } while(*b < 0); x ^= 1; }
|
||||
*a-- = *b;
|
||||
if(b <= buf) { *buf = t; break; }
|
||||
*b-- = *a;
|
||||
if(*b < 0) { p1 = PA + ~*b; x |= 1; }
|
||||
else { p1 = PA + *b; }
|
||||
} else if(r < 0) {
|
||||
if(x & 2) { do { *a-- = *c, *c-- = *a; } while(*c < 0); x ^= 2; }
|
||||
*a-- = *c, *c-- = *a;
|
||||
if(c < first) {
|
||||
while(buf < b) { *a-- = *b, *b-- = *a; }
|
||||
*a = *b, *b = t;
|
||||
break;
|
||||
}
|
||||
if(*c < 0) { p2 = PA + ~*c; x |= 2; }
|
||||
else { p2 = PA + *c; }
|
||||
} else {
|
||||
if(x & 1) { do { *a-- = *b, *b-- = *a; } while(*b < 0); x ^= 1; }
|
||||
*a-- = ~*b;
|
||||
if(b <= buf) { *buf = t; break; }
|
||||
*b-- = *a;
|
||||
if(x & 2) { do { *a-- = *c, *c-- = *a; } while(*c < 0); x ^= 2; }
|
||||
*a-- = *c, *c-- = *a;
|
||||
if(c < first) {
|
||||
while(buf < b) { *a-- = *b, *b-- = *a; }
|
||||
*a = *b, *b = t;
|
||||
break;
|
||||
}
|
||||
if(*b < 0) { p1 = PA + ~*b; x |= 1; }
|
||||
else { p1 = PA + *b; }
|
||||
if(*c < 0) { p2 = PA + ~*c; x |= 2; }
|
||||
else { p2 = PA + *c; }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* D&C based merge. */
|
||||
static
|
||||
void
|
||||
ss_swapmerge(const unsigned char *T, const int *PA,
|
||||
int *first, int *middle, int *last,
|
||||
int *buf, int bufsize, int depth) {
|
||||
#define STACK_SIZE SS_SMERGE_STACKSIZE
|
||||
#define GETIDX(a) ((0 <= (a)) ? (a) : (~(a)))
|
||||
#define MERGE_CHECK(a, b, c)\
|
||||
do {\
|
||||
if(((c) & 1) ||\
|
||||
(((c) & 2) && (ss_compare(T, PA + GETIDX(*((a) - 1)), PA + *(a), depth) == 0))) {\
|
||||
*(a) = ~*(a);\
|
||||
}\
|
||||
if(((c) & 4) && ((ss_compare(T, PA + GETIDX(*((b) - 1)), PA + *(b), depth) == 0))) {\
|
||||
*(b) = ~*(b);\
|
||||
}\
|
||||
} while(0)
|
||||
struct { int *a, *b, *c; int d; } stack[STACK_SIZE];
|
||||
int *l, *r, *lm, *rm;
|
||||
int m, len, half;
|
||||
int ssize;
|
||||
int check, next;
|
||||
|
||||
for(check = 0, ssize = 0;;) {
|
||||
if((last - middle) <= bufsize) {
|
||||
if((first < middle) && (middle < last)) {
|
||||
ss_mergebackward(T, PA, first, middle, last, buf, depth);
|
||||
}
|
||||
MERGE_CHECK(first, last, check);
|
||||
STACK_POP(first, middle, last, check);
|
||||
continue;
|
||||
}
|
||||
|
||||
if((middle - first) <= bufsize) {
|
||||
if(first < middle) {
|
||||
ss_mergeforward(T, PA, first, middle, last, buf, depth);
|
||||
}
|
||||
MERGE_CHECK(first, last, check);
|
||||
STACK_POP(first, middle, last, check);
|
||||
continue;
|
||||
}
|
||||
|
||||
for(m = 0, len = MIN(middle - first, last - middle), half = len >> 1;
|
||||
0 < len;
|
||||
len = half, half >>= 1) {
|
||||
if(ss_compare(T, PA + GETIDX(*(middle + m + half)),
|
||||
PA + GETIDX(*(middle - m - half - 1)), depth) < 0) {
|
||||
m += half + 1;
|
||||
half -= (len & 1) ^ 1;
|
||||
}
|
||||
}
|
||||
|
||||
if(0 < m) {
|
||||
lm = middle - m, rm = middle + m;
|
||||
ss_blockswap(lm, middle, m);
|
||||
l = r = middle, next = 0;
|
||||
if(rm < last) {
|
||||
if(*rm < 0) {
|
||||
*rm = ~*rm;
|
||||
if(first < lm) { for(; *--l < 0;) { } next |= 4; }
|
||||
next |= 1;
|
||||
} else if(first < lm) {
|
||||
for(; *r < 0; ++r) { }
|
||||
next |= 2;
|
||||
}
|
||||
}
|
||||
|
||||
if((l - first) <= (last - r)) {
|
||||
STACK_PUSH(r, rm, last, (next & 3) | (check & 4));
|
||||
middle = lm, last = l, check = (check & 3) | (next & 4);
|
||||
} else {
|
||||
if((next & 2) && (r == middle)) { next ^= 6; }
|
||||
STACK_PUSH(first, lm, l, (check & 3) | (next & 4));
|
||||
first = r, middle = rm, check = (next & 3) | (check & 4);
|
||||
}
|
||||
} else {
|
||||
if(ss_compare(T, PA + GETIDX(*(middle - 1)), PA + *middle, depth) == 0) {
|
||||
*middle = ~*middle;
|
||||
}
|
||||
MERGE_CHECK(first, last, check);
|
||||
STACK_POP(first, middle, last, check);
|
||||
}
|
||||
}
|
||||
#undef STACK_SIZE
|
||||
}
|
||||
|
||||
#endif /* SS_BLOCKSIZE != 0 */
|
||||
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
/* Substring sort */
|
||||
static
|
||||
void
|
||||
sssort(const unsigned char *T, const int *PA,
|
||||
int *first, int *last,
|
||||
int *buf, int bufsize,
|
||||
int depth, int n, int lastsuffix) {
|
||||
int *a;
|
||||
#if SS_BLOCKSIZE != 0
|
||||
int *b, *middle, *curbuf;
|
||||
int j, k, curbufsize, limit;
|
||||
#endif
|
||||
int i;
|
||||
|
||||
if(lastsuffix != 0) { ++first; }
|
||||
|
||||
#if SS_BLOCKSIZE == 0
|
||||
ss_mintrosort(T, PA, first, last, depth);
|
||||
#else
|
||||
if((bufsize < SS_BLOCKSIZE) &&
|
||||
(bufsize < (last - first)) &&
|
||||
(bufsize < (limit = ss_isqrt(last - first)))) {
|
||||
if(SS_BLOCKSIZE < limit) { limit = SS_BLOCKSIZE; }
|
||||
buf = middle = last - limit, bufsize = limit;
|
||||
} else {
|
||||
middle = last, limit = 0;
|
||||
}
|
||||
for(a = first, i = 0; SS_BLOCKSIZE < (middle - a); a += SS_BLOCKSIZE, ++i) {
|
||||
#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE
|
||||
ss_mintrosort(T, PA, a, a + SS_BLOCKSIZE, depth);
|
||||
#elif 1 < SS_BLOCKSIZE
|
||||
ss_insertionsort(T, PA, a, a + SS_BLOCKSIZE, depth);
|
||||
#endif
|
||||
curbufsize = last - (a + SS_BLOCKSIZE);
|
||||
curbuf = a + SS_BLOCKSIZE;
|
||||
if(curbufsize <= bufsize) { curbufsize = bufsize, curbuf = buf; }
|
||||
for(b = a, k = SS_BLOCKSIZE, j = i; j & 1; b -= k, k <<= 1, j >>= 1) {
|
||||
ss_swapmerge(T, PA, b - k, b, b + k, curbuf, curbufsize, depth);
|
||||
}
|
||||
}
|
||||
#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE
|
||||
ss_mintrosort(T, PA, a, middle, depth);
|
||||
#elif 1 < SS_BLOCKSIZE
|
||||
ss_insertionsort(T, PA, a, middle, depth);
|
||||
#endif
|
||||
for(k = SS_BLOCKSIZE; i != 0; k <<= 1, i >>= 1) {
|
||||
if(i & 1) {
|
||||
ss_swapmerge(T, PA, a - k, a, middle, buf, bufsize, depth);
|
||||
a -= k;
|
||||
}
|
||||
}
|
||||
if(limit != 0) {
|
||||
#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE
|
||||
ss_mintrosort(T, PA, middle, last, depth);
|
||||
#elif 1 < SS_BLOCKSIZE
|
||||
ss_insertionsort(T, PA, middle, last, depth);
|
||||
#endif
|
||||
ss_inplacemerge(T, PA, first, middle, last, depth);
|
||||
}
|
||||
#endif
|
||||
|
||||
if(lastsuffix != 0) {
|
||||
/* Insert last type B* suffix. */
|
||||
int PAi[2]; PAi[0] = PA[*(first - 1)], PAi[1] = n - 2;
|
||||
for(a = first, i = *(first - 1);
|
||||
(a < last) && ((*a < 0) || (0 < ss_compare(T, &(PAi[0]), PA + *a, depth)));
|
||||
++a) {
|
||||
*(a - 1) = *a;
|
||||
}
|
||||
*(a - 1) = i;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
static INLINE
|
||||
int
|
||||
tr_ilg(int n) {
|
||||
return (n & 0xffff0000) ?
|
||||
((n & 0xff000000) ?
|
||||
24 + lg_table[(n >> 24) & 0xff] :
|
||||
16 + lg_table[(n >> 16) & 0xff]) :
|
||||
((n & 0x0000ff00) ?
|
||||
8 + lg_table[(n >> 8) & 0xff] :
|
||||
0 + lg_table[(n >> 0) & 0xff]);
|
||||
}
|
||||
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
/* Simple insertionsort for small size groups. */
|
||||
static
|
||||
void
|
||||
tr_insertionsort(const int *ISAd, int *first, int *last) {
|
||||
int *a, *b;
|
||||
int t, r;
|
||||
|
||||
for(a = first + 1; a < last; ++a) {
|
||||
for(t = *a, b = a - 1; 0 > (r = ISAd[t] - ISAd[*b]);) {
|
||||
do { *(b + 1) = *b; } while((first <= --b) && (*b < 0));
|
||||
if(b < first) { break; }
|
||||
}
|
||||
if(r == 0) { *b = ~*b; }
|
||||
*(b + 1) = t;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
static INLINE
|
||||
void
|
||||
tr_fixdown(const int *ISAd, int *SA, int i, int size) {
|
||||
int j, k;
|
||||
int v;
|
||||
int c, d, e;
|
||||
|
||||
for(v = SA[i], c = ISAd[v]; (j = 2 * i + 1) < size; SA[i] = SA[k], i = k) {
|
||||
d = ISAd[SA[k = j++]];
|
||||
if(d < (e = ISAd[SA[j]])) { k = j; d = e; }
|
||||
if(d <= c) { break; }
|
||||
}
|
||||
SA[i] = v;
|
||||
}
|
||||
|
||||
/* Simple top-down heapsort. */
|
||||
static
|
||||
void
|
||||
tr_heapsort(const int *ISAd, int *SA, int size) {
|
||||
int i, m;
|
||||
int t;
|
||||
|
||||
m = size;
|
||||
if((size % 2) == 0) {
|
||||
m--;
|
||||
if(ISAd[SA[m / 2]] < ISAd[SA[m]]) { SWAP(SA[m], SA[m / 2]); }
|
||||
}
|
||||
|
||||
for(i = m / 2 - 1; 0 <= i; --i) { tr_fixdown(ISAd, SA, i, m); }
|
||||
if((size % 2) == 0) { SWAP(SA[0], SA[m]); tr_fixdown(ISAd, SA, 0, m); }
|
||||
for(i = m - 1; 0 < i; --i) {
|
||||
t = SA[0], SA[0] = SA[i];
|
||||
tr_fixdown(ISAd, SA, 0, i);
|
||||
SA[i] = t;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
/* Returns the median of three elements. */
|
||||
static INLINE
|
||||
int *
|
||||
tr_median3(const int *ISAd, int *v1, int *v2, int *v3) {
|
||||
int *t;
|
||||
if(ISAd[*v1] > ISAd[*v2]) { SWAP(v1, v2); }
|
||||
if(ISAd[*v2] > ISAd[*v3]) {
|
||||
if(ISAd[*v1] > ISAd[*v3]) { return v1; }
|
||||
else { return v3; }
|
||||
}
|
||||
return v2;
|
||||
}
|
||||
|
||||
/* Returns the median of five elements. */
|
||||
static INLINE
|
||||
int *
|
||||
tr_median5(const int *ISAd,
|
||||
int *v1, int *v2, int *v3, int *v4, int *v5) {
|
||||
int *t;
|
||||
if(ISAd[*v2] > ISAd[*v3]) { SWAP(v2, v3); }
|
||||
if(ISAd[*v4] > ISAd[*v5]) { SWAP(v4, v5); }
|
||||
if(ISAd[*v2] > ISAd[*v4]) { SWAP(v2, v4); SWAP(v3, v5); }
|
||||
if(ISAd[*v1] > ISAd[*v3]) { SWAP(v1, v3); }
|
||||
if(ISAd[*v1] > ISAd[*v4]) { SWAP(v1, v4); SWAP(v3, v5); }
|
||||
if(ISAd[*v3] > ISAd[*v4]) { return v4; }
|
||||
return v3;
|
||||
}
|
||||
|
||||
/* Returns the pivot element. */
|
||||
static INLINE
|
||||
int *
|
||||
tr_pivot(const int *ISAd, int *first, int *last) {
|
||||
int *middle;
|
||||
int t;
|
||||
|
||||
t = last - first;
|
||||
middle = first + t / 2;
|
||||
|
||||
if(t <= 512) {
|
||||
if(t <= 32) {
|
||||
return tr_median3(ISAd, first, middle, last - 1);
|
||||
} else {
|
||||
t >>= 2;
|
||||
return tr_median5(ISAd, first, first + t, middle, last - 1 - t, last - 1);
|
||||
}
|
||||
}
|
||||
t >>= 3;
|
||||
first = tr_median3(ISAd, first, first + t, first + (t << 1));
|
||||
middle = tr_median3(ISAd, middle - t, middle, middle + t);
|
||||
last = tr_median3(ISAd, last - 1 - (t << 1), last - 1 - t, last - 1);
|
||||
return tr_median3(ISAd, first, middle, last);
|
||||
}
|
||||
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
typedef struct _trbudget_t trbudget_t;
|
||||
struct _trbudget_t {
|
||||
int chance;
|
||||
int remain;
|
||||
int incval;
|
||||
int count;
|
||||
};
|
||||
|
||||
static INLINE
|
||||
void
|
||||
trbudget_init(trbudget_t *budget, int chance, int incval) {
|
||||
budget->chance = chance;
|
||||
budget->remain = budget->incval = incval;
|
||||
}
|
||||
|
||||
static INLINE
|
||||
int
|
||||
trbudget_check(trbudget_t *budget, int size) {
|
||||
if(size <= budget->remain) { budget->remain -= size; return 1; }
|
||||
if(budget->chance == 0) { budget->count += size; return 0; }
|
||||
budget->remain += budget->incval - size;
|
||||
budget->chance -= 1;
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
static INLINE
|
||||
void
|
||||
tr_partition(const int *ISAd,
|
||||
int *first, int *middle, int *last,
|
||||
int **pa, int **pb, int v) {
|
||||
int *a, *b, *c, *d, *e, *f;
|
||||
int t, s;
|
||||
int x = 0;
|
||||
|
||||
for(b = middle - 1; (++b < last) && ((x = ISAd[*b]) == v);) { }
|
||||
if(((a = b) < last) && (x < v)) {
|
||||
for(; (++b < last) && ((x = ISAd[*b]) <= v);) {
|
||||
if(x == v) { SWAP(*b, *a); ++a; }
|
||||
}
|
||||
}
|
||||
for(c = last; (b < --c) && ((x = ISAd[*c]) == v);) { }
|
||||
if((b < (d = c)) && (x > v)) {
|
||||
for(; (b < --c) && ((x = ISAd[*c]) >= v);) {
|
||||
if(x == v) { SWAP(*c, *d); --d; }
|
||||
}
|
||||
}
|
||||
for(; b < c;) {
|
||||
SWAP(*b, *c);
|
||||
for(; (++b < c) && ((x = ISAd[*b]) <= v);) {
|
||||
if(x == v) { SWAP(*b, *a); ++a; }
|
||||
}
|
||||
for(; (b < --c) && ((x = ISAd[*c]) >= v);) {
|
||||
if(x == v) { SWAP(*c, *d); --d; }
|
||||
}
|
||||
}
|
||||
|
||||
if(a <= d) {
|
||||
c = b - 1;
|
||||
if((s = a - first) > (t = b - a)) { s = t; }
|
||||
for(e = first, f = b - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); }
|
||||
if((s = d - c) > (t = last - d - 1)) { s = t; }
|
||||
for(e = b, f = last - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); }
|
||||
first += (b - a), last -= (d - c);
|
||||
}
|
||||
*pa = first, *pb = last;
|
||||
}
|
||||
|
||||
static
|
||||
void
|
||||
tr_copy(int *ISA, const int *SA,
|
||||
int *first, int *a, int *b, int *last,
|
||||
int depth) {
|
||||
/* sort suffixes of middle partition
|
||||
by using sorted order of suffixes of left and right partition. */
|
||||
int *c, *d, *e;
|
||||
int s, v;
|
||||
|
||||
v = b - SA - 1;
|
||||
for(c = first, d = a - 1; c <= d; ++c) {
|
||||
if((0 <= (s = *c - depth)) && (ISA[s] == v)) {
|
||||
*++d = s;
|
||||
ISA[s] = d - SA;
|
||||
}
|
||||
}
|
||||
for(c = last - 1, e = d + 1, d = b; e < d; --c) {
|
||||
if((0 <= (s = *c - depth)) && (ISA[s] == v)) {
|
||||
*--d = s;
|
||||
ISA[s] = d - SA;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static
|
||||
void
|
||||
tr_partialcopy(int *ISA, const int *SA,
|
||||
int *first, int *a, int *b, int *last,
|
||||
int depth) {
|
||||
int *c, *d, *e;
|
||||
int s, v;
|
||||
int rank, lastrank, newrank = -1;
|
||||
|
||||
v = b - SA - 1;
|
||||
lastrank = -1;
|
||||
for(c = first, d = a - 1; c <= d; ++c) {
|
||||
if((0 <= (s = *c - depth)) && (ISA[s] == v)) {
|
||||
*++d = s;
|
||||
rank = ISA[s + depth];
|
||||
if(lastrank != rank) { lastrank = rank; newrank = d - SA; }
|
||||
ISA[s] = newrank;
|
||||
}
|
||||
}
|
||||
|
||||
lastrank = -1;
|
||||
for(e = d; first <= e; --e) {
|
||||
rank = ISA[*e];
|
||||
if(lastrank != rank) { lastrank = rank; newrank = e - SA; }
|
||||
if(newrank != rank) { ISA[*e] = newrank; }
|
||||
}
|
||||
|
||||
lastrank = -1;
|
||||
for(c = last - 1, e = d + 1, d = b; e < d; --c) {
|
||||
if((0 <= (s = *c - depth)) && (ISA[s] == v)) {
|
||||
*--d = s;
|
||||
rank = ISA[s + depth];
|
||||
if(lastrank != rank) { lastrank = rank; newrank = d - SA; }
|
||||
ISA[s] = newrank;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static
|
||||
void
|
||||
tr_introsort(int *ISA, const int *ISAd,
|
||||
int *SA, int *first, int *last,
|
||||
trbudget_t *budget) {
|
||||
#define STACK_SIZE TR_STACKSIZE
|
||||
struct { const int *a; int *b, *c; int d, e; }stack[STACK_SIZE];
|
||||
int *a, *b, *c;
|
||||
int t;
|
||||
int v, x = 0;
|
||||
int incr = ISAd - ISA;
|
||||
int limit, next;
|
||||
int ssize, trlink = -1;
|
||||
|
||||
for(ssize = 0, limit = tr_ilg(last - first);;) {
|
||||
|
||||
if(limit < 0) {
|
||||
if(limit == -1) {
|
||||
/* tandem repeat partition */
|
||||
tr_partition(ISAd - incr, first, first, last, &a, &b, last - SA - 1);
|
||||
|
||||
/* update ranks */
|
||||
if(a < last) {
|
||||
for(c = first, v = a - SA - 1; c < a; ++c) { ISA[*c] = v; }
|
||||
}
|
||||
if(b < last) {
|
||||
for(c = a, v = b - SA - 1; c < b; ++c) { ISA[*c] = v; }
|
||||
}
|
||||
|
||||
/* push */
|
||||
if(1 < (b - a)) {
|
||||
STACK_PUSH5(NULL, a, b, 0, 0);
|
||||
STACK_PUSH5(ISAd - incr, first, last, -2, trlink);
|
||||
trlink = ssize - 2;
|
||||
}
|
||||
if((a - first) <= (last - b)) {
|
||||
if(1 < (a - first)) {
|
||||
STACK_PUSH5(ISAd, b, last, tr_ilg(last - b), trlink);
|
||||
last = a, limit = tr_ilg(a - first);
|
||||
} else if(1 < (last - b)) {
|
||||
first = b, limit = tr_ilg(last - b);
|
||||
} else {
|
||||
STACK_POP5(ISAd, first, last, limit, trlink);
|
||||
}
|
||||
} else {
|
||||
if(1 < (last - b)) {
|
||||
STACK_PUSH5(ISAd, first, a, tr_ilg(a - first), trlink);
|
||||
first = b, limit = tr_ilg(last - b);
|
||||
} else if(1 < (a - first)) {
|
||||
last = a, limit = tr_ilg(a - first);
|
||||
} else {
|
||||
STACK_POP5(ISAd, first, last, limit, trlink);
|
||||
}
|
||||
}
|
||||
} else if(limit == -2) {
|
||||
/* tandem repeat copy */
|
||||
a = stack[--ssize].b, b = stack[ssize].c;
|
||||
if(stack[ssize].d == 0) {
|
||||
tr_copy(ISA, SA, first, a, b, last, ISAd - ISA);
|
||||
} else {
|
||||
if(0 <= trlink) { stack[trlink].d = -1; }
|
||||
tr_partialcopy(ISA, SA, first, a, b, last, ISAd - ISA);
|
||||
}
|
||||
STACK_POP5(ISAd, first, last, limit, trlink);
|
||||
} else {
|
||||
/* sorted partition */
|
||||
if(0 <= *first) {
|
||||
a = first;
|
||||
do { ISA[*a] = a - SA; } while((++a < last) && (0 <= *a));
|
||||
first = a;
|
||||
}
|
||||
if(first < last) {
|
||||
a = first; do { *a = ~*a; } while(*++a < 0);
|
||||
next = (ISA[*a] != ISAd[*a]) ? tr_ilg(a - first + 1) : -1;
|
||||
if(++a < last) { for(b = first, v = a - SA - 1; b < a; ++b) { ISA[*b] = v; } }
|
||||
|
||||
/* push */
|
||||
if(trbudget_check(budget, a - first)) {
|
||||
if((a - first) <= (last - a)) {
|
||||
STACK_PUSH5(ISAd, a, last, -3, trlink);
|
||||
ISAd += incr, last = a, limit = next;
|
||||
} else {
|
||||
if(1 < (last - a)) {
|
||||
STACK_PUSH5(ISAd + incr, first, a, next, trlink);
|
||||
first = a, limit = -3;
|
||||
} else {
|
||||
ISAd += incr, last = a, limit = next;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if(0 <= trlink) { stack[trlink].d = -1; }
|
||||
if(1 < (last - a)) {
|
||||
first = a, limit = -3;
|
||||
} else {
|
||||
STACK_POP5(ISAd, first, last, limit, trlink);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
STACK_POP5(ISAd, first, last, limit, trlink);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if((last - first) <= TR_INSERTIONSORT_THRESHOLD) {
|
||||
tr_insertionsort(ISAd, first, last);
|
||||
limit = -3;
|
||||
continue;
|
||||
}
|
||||
|
||||
if(limit-- == 0) {
|
||||
tr_heapsort(ISAd, first, last - first);
|
||||
for(a = last - 1; first < a; a = b) {
|
||||
for(x = ISAd[*a], b = a - 1; (first <= b) && (ISAd[*b] == x); --b) { *b = ~*b; }
|
||||
}
|
||||
limit = -3;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* choose pivot */
|
||||
a = tr_pivot(ISAd, first, last);
|
||||
SWAP(*first, *a);
|
||||
v = ISAd[*first];
|
||||
|
||||
/* partition */
|
||||
tr_partition(ISAd, first, first + 1, last, &a, &b, v);
|
||||
if((last - first) != (b - a)) {
|
||||
next = (ISA[*a] != v) ? tr_ilg(b - a) : -1;
|
||||
|
||||
/* update ranks */
|
||||
for(c = first, v = a - SA - 1; c < a; ++c) { ISA[*c] = v; }
|
||||
if(b < last) { for(c = a, v = b - SA - 1; c < b; ++c) { ISA[*c] = v; } }
|
||||
|
||||
/* push */
|
||||
if((1 < (b - a)) && (trbudget_check(budget, b - a))) {
|
||||
if((a - first) <= (last - b)) {
|
||||
if((last - b) <= (b - a)) {
|
||||
if(1 < (a - first)) {
|
||||
STACK_PUSH5(ISAd + incr, a, b, next, trlink);
|
||||
STACK_PUSH5(ISAd, b, last, limit, trlink);
|
||||
last = a;
|
||||
} else if(1 < (last - b)) {
|
||||
STACK_PUSH5(ISAd + incr, a, b, next, trlink);
|
||||
first = b;
|
||||
} else {
|
||||
ISAd += incr, first = a, last = b, limit = next;
|
||||
}
|
||||
} else if((a - first) <= (b - a)) {
|
||||
if(1 < (a - first)) {
|
||||
STACK_PUSH5(ISAd, b, last, limit, trlink);
|
||||
STACK_PUSH5(ISAd + incr, a, b, next, trlink);
|
||||
last = a;
|
||||
} else {
|
||||
STACK_PUSH5(ISAd, b, last, limit, trlink);
|
||||
ISAd += incr, first = a, last = b, limit = next;
|
||||
}
|
||||
} else {
|
||||
STACK_PUSH5(ISAd, b, last, limit, trlink);
|
||||
STACK_PUSH5(ISAd, first, a, limit, trlink);
|
||||
ISAd += incr, first = a, last = b, limit = next;
|
||||
}
|
||||
} else {
|
||||
if((a - first) <= (b - a)) {
|
||||
if(1 < (last - b)) {
|
||||
STACK_PUSH5(ISAd + incr, a, b, next, trlink);
|
||||
STACK_PUSH5(ISAd, first, a, limit, trlink);
|
||||
first = b;
|
||||
} else if(1 < (a - first)) {
|
||||
STACK_PUSH5(ISAd + incr, a, b, next, trlink);
|
||||
last = a;
|
||||
} else {
|
||||
ISAd += incr, first = a, last = b, limit = next;
|
||||
}
|
||||
} else if((last - b) <= (b - a)) {
|
||||
if(1 < (last - b)) {
|
||||
STACK_PUSH5(ISAd, first, a, limit, trlink);
|
||||
STACK_PUSH5(ISAd + incr, a, b, next, trlink);
|
||||
first = b;
|
||||
} else {
|
||||
STACK_PUSH5(ISAd, first, a, limit, trlink);
|
||||
ISAd += incr, first = a, last = b, limit = next;
|
||||
}
|
||||
} else {
|
||||
STACK_PUSH5(ISAd, first, a, limit, trlink);
|
||||
STACK_PUSH5(ISAd, b, last, limit, trlink);
|
||||
ISAd += incr, first = a, last = b, limit = next;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if((1 < (b - a)) && (0 <= trlink)) { stack[trlink].d = -1; }
|
||||
if((a - first) <= (last - b)) {
|
||||
if(1 < (a - first)) {
|
||||
STACK_PUSH5(ISAd, b, last, limit, trlink);
|
||||
last = a;
|
||||
} else if(1 < (last - b)) {
|
||||
first = b;
|
||||
} else {
|
||||
STACK_POP5(ISAd, first, last, limit, trlink);
|
||||
}
|
||||
} else {
|
||||
if(1 < (last - b)) {
|
||||
STACK_PUSH5(ISAd, first, a, limit, trlink);
|
||||
first = b;
|
||||
} else if(1 < (a - first)) {
|
||||
last = a;
|
||||
} else {
|
||||
STACK_POP5(ISAd, first, last, limit, trlink);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if(trbudget_check(budget, last - first)) {
|
||||
limit = tr_ilg(last - first), ISAd += incr;
|
||||
} else {
|
||||
if(0 <= trlink) { stack[trlink].d = -1; }
|
||||
STACK_POP5(ISAd, first, last, limit, trlink);
|
||||
}
|
||||
}
|
||||
}
|
||||
#undef STACK_SIZE
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
/* Tandem repeat sort */
|
||||
static
|
||||
void
|
||||
trsort(int *ISA, int *SA, int n, int depth) {
|
||||
int *ISAd;
|
||||
int *first, *last;
|
||||
trbudget_t budget;
|
||||
int t, skip, unsorted;
|
||||
|
||||
trbudget_init(&budget, tr_ilg(n) * 2 / 3, n);
|
||||
/* trbudget_init(&budget, tr_ilg(n) * 3 / 4, n); */
|
||||
for(ISAd = ISA + depth; -n < *SA; ISAd += ISAd - ISA) {
|
||||
first = SA;
|
||||
skip = 0;
|
||||
unsorted = 0;
|
||||
do {
|
||||
if((t = *first) < 0) { first -= t; skip += t; }
|
||||
else {
|
||||
if(skip != 0) { *(first + skip) = skip; skip = 0; }
|
||||
last = SA + ISA[t] + 1;
|
||||
if(1 < (last - first)) {
|
||||
budget.count = 0;
|
||||
tr_introsort(ISA, ISAd, SA, first, last, &budget);
|
||||
if(budget.count != 0) { unsorted += budget.count; }
|
||||
else { skip = first - last; }
|
||||
} else if((last - first) == 1) {
|
||||
skip = -1;
|
||||
}
|
||||
first = last;
|
||||
}
|
||||
} while(first < (SA + n));
|
||||
if(skip != 0) { *(first + skip) = skip; }
|
||||
if(unsorted == 0) { break; }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
/* Sorts suffixes of type B*. */
|
||||
static
|
||||
int
|
||||
sort_typeBstar(const unsigned char *T, int *SA,
|
||||
int *bucket_A, int *bucket_B,
|
||||
int n, int openMP) {
|
||||
int *PAb, *ISAb, *buf;
|
||||
#ifdef LIBBSC_OPENMP
|
||||
int *curbuf;
|
||||
int l;
|
||||
#endif
|
||||
int i, j, k, t, m, bufsize;
|
||||
int c0, c1;
|
||||
#ifdef LIBBSC_OPENMP
|
||||
int d0, d1;
|
||||
#endif
|
||||
(void)openMP;
|
||||
|
||||
/* Initialize bucket arrays. */
|
||||
for(i = 0; i < BUCKET_A_SIZE; ++i) { bucket_A[i] = 0; }
|
||||
for(i = 0; i < BUCKET_B_SIZE; ++i) { bucket_B[i] = 0; }
|
||||
|
||||
/* Count the number of occurrences of the first one or two characters of each
|
||||
type A, B and B* suffix. Moreover, store the beginning position of all
|
||||
type B* suffixes into the array SA. */
|
||||
for(i = n - 1, m = n, c0 = T[n - 1]; 0 <= i;) {
|
||||
/* type A suffix. */
|
||||
do { ++BUCKET_A(c1 = c0); } while((0 <= --i) && ((c0 = T[i]) >= c1));
|
||||
if(0 <= i) {
|
||||
/* type B* suffix. */
|
||||
++BUCKET_BSTAR(c0, c1);
|
||||
SA[--m] = i;
|
||||
/* type B suffix. */
|
||||
for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) <= c1); --i, c1 = c0) {
|
||||
++BUCKET_B(c0, c1);
|
||||
}
|
||||
}
|
||||
}
|
||||
m = n - m;
|
||||
/*
|
||||
note:
|
||||
A type B* suffix is lexicographically smaller than a type B suffix that
|
||||
begins with the same first two characters.
|
||||
*/
|
||||
|
||||
/* Calculate the index of start/end point of each bucket. */
|
||||
for(c0 = 0, i = 0, j = 0; c0 < ALPHABET_SIZE; ++c0) {
|
||||
t = i + BUCKET_A(c0);
|
||||
BUCKET_A(c0) = i + j; /* start point */
|
||||
i = t + BUCKET_B(c0, c0);
|
||||
for(c1 = c0 + 1; c1 < ALPHABET_SIZE; ++c1) {
|
||||
j += BUCKET_BSTAR(c0, c1);
|
||||
BUCKET_BSTAR(c0, c1) = j; /* end point */
|
||||
i += BUCKET_B(c0, c1);
|
||||
}
|
||||
}
|
||||
|
||||
if(0 < m) {
|
||||
/* Sort the type B* suffixes by their first two characters. */
|
||||
PAb = SA + n - m; ISAb = SA + m;
|
||||
for(i = m - 2; 0 <= i; --i) {
|
||||
t = PAb[i], c0 = T[t], c1 = T[t + 1];
|
||||
SA[--BUCKET_BSTAR(c0, c1)] = i;
|
||||
}
|
||||
t = PAb[m - 1], c0 = T[t], c1 = T[t + 1];
|
||||
SA[--BUCKET_BSTAR(c0, c1)] = m - 1;
|
||||
|
||||
/* Sort the type B* substrings using sssort. */
|
||||
#ifdef LIBBSC_OPENMP
|
||||
if (openMP)
|
||||
{
|
||||
buf = SA + m;
|
||||
c0 = ALPHABET_SIZE - 2, c1 = ALPHABET_SIZE - 1, j = m;
|
||||
#pragma omp parallel default(shared) private(bufsize, curbuf, k, l, d0, d1)
|
||||
{
|
||||
bufsize = (n - (2 * m)) / omp_get_num_threads();
|
||||
curbuf = buf + omp_get_thread_num() * bufsize;
|
||||
k = 0;
|
||||
for(;;) {
|
||||
#pragma omp critical(sssort_lock)
|
||||
{
|
||||
if(0 < (l = j)) {
|
||||
d0 = c0, d1 = c1;
|
||||
do {
|
||||
k = BUCKET_BSTAR(d0, d1);
|
||||
if(--d1 <= d0) {
|
||||
d1 = ALPHABET_SIZE - 1;
|
||||
if(--d0 < 0) { break; }
|
||||
}
|
||||
} while(((l - k) <= 1) && (0 < (l = k)));
|
||||
c0 = d0, c1 = d1, j = k;
|
||||
}
|
||||
}
|
||||
if(l == 0) { break; }
|
||||
sssort(T, PAb, SA + k, SA + l,
|
||||
curbuf, bufsize, 2, n, *(SA + k) == (m - 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
buf = SA + m, bufsize = n - (2 * m);
|
||||
for(c0 = ALPHABET_SIZE - 2, j = m; 0 < j; --c0) {
|
||||
for(c1 = ALPHABET_SIZE - 1; c0 < c1; j = i, --c1) {
|
||||
i = BUCKET_BSTAR(c0, c1);
|
||||
if(1 < (j - i)) {
|
||||
sssort(T, PAb, SA + i, SA + j,
|
||||
buf, bufsize, 2, n, *(SA + i) == (m - 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#else
|
||||
buf = SA + m, bufsize = n - (2 * m);
|
||||
for(c0 = ALPHABET_SIZE - 2, j = m; 0 < j; --c0) {
|
||||
for(c1 = ALPHABET_SIZE - 1; c0 < c1; j = i, --c1) {
|
||||
i = BUCKET_BSTAR(c0, c1);
|
||||
if(1 < (j - i)) {
|
||||
sssort(T, PAb, SA + i, SA + j,
|
||||
buf, bufsize, 2, n, *(SA + i) == (m - 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Compute ranks of type B* substrings. */
|
||||
for(i = m - 1; 0 <= i; --i) {
|
||||
if(0 <= SA[i]) {
|
||||
j = i;
|
||||
do { ISAb[SA[i]] = i; } while((0 <= --i) && (0 <= SA[i]));
|
||||
SA[i + 1] = i - j;
|
||||
if(i <= 0) { break; }
|
||||
}
|
||||
j = i;
|
||||
do { ISAb[SA[i] = ~SA[i]] = j; } while(SA[--i] < 0);
|
||||
ISAb[SA[i]] = j;
|
||||
}
|
||||
|
||||
/* Construct the inverse suffix array of type B* suffixes using trsort. */
|
||||
trsort(ISAb, SA, m, 1);
|
||||
|
||||
/* Set the sorted order of type B* suffixes. */
|
||||
for(i = n - 1, j = m, c0 = T[n - 1]; 0 <= i;) {
|
||||
for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) >= c1); --i, c1 = c0) { }
|
||||
if(0 <= i) {
|
||||
t = i;
|
||||
for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) <= c1); --i, c1 = c0) { }
|
||||
SA[ISAb[--j]] = ((t == 0) || (1 < (t - i))) ? t : ~t;
|
||||
}
|
||||
}
|
||||
|
||||
/* Calculate the index of start/end point of each bucket. */
|
||||
BUCKET_B(ALPHABET_SIZE - 1, ALPHABET_SIZE - 1) = n; /* end point */
|
||||
for(c0 = ALPHABET_SIZE - 2, k = m - 1; 0 <= c0; --c0) {
|
||||
i = BUCKET_A(c0 + 1) - 1;
|
||||
for(c1 = ALPHABET_SIZE - 1; c0 < c1; --c1) {
|
||||
t = i - BUCKET_B(c0, c1);
|
||||
BUCKET_B(c0, c1) = i; /* end point */
|
||||
|
||||
/* Move all type B* suffixes to the correct position. */
|
||||
for(i = t, j = BUCKET_BSTAR(c0, c1);
|
||||
j <= k;
|
||||
--i, --k) { SA[i] = SA[k]; }
|
||||
}
|
||||
BUCKET_BSTAR(c0, c0 + 1) = i - BUCKET_B(c0, c0) + 1; /* start point */
|
||||
BUCKET_B(c0, c0) = i; /* end point */
|
||||
}
|
||||
}
|
||||
|
||||
return m;
|
||||
}
|
||||
|
||||
/* Constructs the suffix array by using the sorted order of type B* suffixes. */
|
||||
static
|
||||
void
|
||||
construct_SA(const unsigned char *T, int *SA,
|
||||
int *bucket_A, int *bucket_B,
|
||||
int n, int m) {
|
||||
int *i, *j, *k;
|
||||
int s;
|
||||
int c0, c1, c2;
|
||||
|
||||
if(0 < m) {
|
||||
/* Construct the sorted order of type B suffixes by using
|
||||
the sorted order of type B* suffixes. */
|
||||
for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) {
|
||||
/* Scan the suffix array from right to left. */
|
||||
for(i = SA + BUCKET_BSTAR(c1, c1 + 1),
|
||||
j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1;
|
||||
i <= j;
|
||||
--j) {
|
||||
if(0 < (s = *j)) {
|
||||
assert(T[s] == c1);
|
||||
assert(((s + 1) < n) && (T[s] <= T[s + 1]));
|
||||
assert(T[s - 1] <= T[s]);
|
||||
*j = ~s;
|
||||
c0 = T[--s];
|
||||
if((0 < s) && (T[s - 1] > c0)) { s = ~s; }
|
||||
if(c0 != c2) {
|
||||
if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
|
||||
k = SA + BUCKET_B(c2 = c0, c1);
|
||||
}
|
||||
assert(k < j); assert(k != NULL);
|
||||
*k-- = s;
|
||||
} else {
|
||||
assert(((s == 0) && (T[s] == c1)) || (s < 0));
|
||||
*j = ~s;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Construct the suffix array by using
|
||||
the sorted order of type B suffixes. */
|
||||
k = SA + BUCKET_A(c2 = T[n - 1]);
|
||||
*k++ = (T[n - 2] < c2) ? ~(n - 1) : (n - 1);
|
||||
/* Scan the suffix array from left to right. */
|
||||
for(i = SA, j = SA + n; i < j; ++i) {
|
||||
if(0 < (s = *i)) {
|
||||
assert(T[s - 1] >= T[s]);
|
||||
c0 = T[--s];
|
||||
if((s == 0) || (T[s - 1] < c0)) { s = ~s; }
|
||||
if(c0 != c2) {
|
||||
BUCKET_A(c2) = k - SA;
|
||||
k = SA + BUCKET_A(c2 = c0);
|
||||
}
|
||||
assert(i < k);
|
||||
*k++ = s;
|
||||
} else {
|
||||
assert(s < 0);
|
||||
*i = ~s;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Constructs the burrows-wheeler transformed string directly
|
||||
by using the sorted order of type B* suffixes. */
|
||||
static
|
||||
int
|
||||
construct_BWT(const unsigned char *T, int *SA,
|
||||
int *bucket_A, int *bucket_B,
|
||||
int n, int m) {
|
||||
int *i, *j, *k, *orig;
|
||||
int s;
|
||||
int c0, c1, c2;
|
||||
|
||||
if(0 < m) {
|
||||
/* Construct the sorted order of type B suffixes by using
|
||||
the sorted order of type B* suffixes. */
|
||||
for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) {
|
||||
/* Scan the suffix array from right to left. */
|
||||
for(i = SA + BUCKET_BSTAR(c1, c1 + 1),
|
||||
j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1;
|
||||
i <= j;
|
||||
--j) {
|
||||
if(0 < (s = *j)) {
|
||||
assert(T[s] == c1);
|
||||
assert(((s + 1) < n) && (T[s] <= T[s + 1]));
|
||||
assert(T[s - 1] <= T[s]);
|
||||
c0 = T[--s];
|
||||
*j = ~((int)c0);
|
||||
if((0 < s) && (T[s - 1] > c0)) { s = ~s; }
|
||||
if(c0 != c2) {
|
||||
if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
|
||||
k = SA + BUCKET_B(c2 = c0, c1);
|
||||
}
|
||||
assert(k < j); assert(k != NULL);
|
||||
*k-- = s;
|
||||
} else if(s != 0) {
|
||||
*j = ~s;
|
||||
#ifndef NDEBUG
|
||||
} else {
|
||||
assert(T[s] == c1);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Construct the BWTed string by using
|
||||
the sorted order of type B suffixes. */
|
||||
k = SA + BUCKET_A(c2 = T[n - 1]);
|
||||
*k++ = (T[n - 2] < c2) ? ~((int)T[n - 2]) : (n - 1);
|
||||
/* Scan the suffix array from left to right. */
|
||||
for(i = SA, j = SA + n, orig = SA; i < j; ++i) {
|
||||
if(0 < (s = *i)) {
|
||||
assert(T[s - 1] >= T[s]);
|
||||
c0 = T[--s];
|
||||
*i = c0;
|
||||
if((0 < s) && (T[s - 1] < c0)) { s = ~((int)T[s - 1]); }
|
||||
if(c0 != c2) {
|
||||
BUCKET_A(c2) = k - SA;
|
||||
k = SA + BUCKET_A(c2 = c0);
|
||||
}
|
||||
assert(i < k);
|
||||
*k++ = s;
|
||||
} else if(s != 0) {
|
||||
*i = ~s;
|
||||
} else {
|
||||
orig = i;
|
||||
}
|
||||
}
|
||||
|
||||
return orig - SA;
|
||||
}
|
||||
|
||||
/* Constructs the burrows-wheeler transformed string directly
|
||||
by using the sorted order of type B* suffixes. */
|
||||
static
|
||||
int
|
||||
construct_BWT_indexes(const unsigned char *T, int *SA,
|
||||
int *bucket_A, int *bucket_B,
|
||||
int n, int m,
|
||||
unsigned char * num_indexes, int * indexes) {
|
||||
int *i, *j, *k, *orig;
|
||||
int s;
|
||||
int c0, c1, c2;
|
||||
|
||||
int mod = n / 8;
|
||||
{
|
||||
mod |= mod >> 1; mod |= mod >> 2;
|
||||
mod |= mod >> 4; mod |= mod >> 8;
|
||||
mod |= mod >> 16; mod >>= 1;
|
||||
|
||||
*num_indexes = (unsigned char)((n - 1) / (mod + 1));
|
||||
}
|
||||
|
||||
if(0 < m) {
|
||||
/* Construct the sorted order of type B suffixes by using
|
||||
the sorted order of type B* suffixes. */
|
||||
for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) {
|
||||
/* Scan the suffix array from right to left. */
|
||||
for(i = SA + BUCKET_BSTAR(c1, c1 + 1),
|
||||
j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1;
|
||||
i <= j;
|
||||
--j) {
|
||||
if(0 < (s = *j)) {
|
||||
assert(T[s] == c1);
|
||||
assert(((s + 1) < n) && (T[s] <= T[s + 1]));
|
||||
assert(T[s - 1] <= T[s]);
|
||||
|
||||
if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = j - SA;
|
||||
|
||||
c0 = T[--s];
|
||||
*j = ~((int)c0);
|
||||
if((0 < s) && (T[s - 1] > c0)) { s = ~s; }
|
||||
if(c0 != c2) {
|
||||
if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
|
||||
k = SA + BUCKET_B(c2 = c0, c1);
|
||||
}
|
||||
assert(k < j); assert(k != NULL);
|
||||
*k-- = s;
|
||||
} else if(s != 0) {
|
||||
*j = ~s;
|
||||
#ifndef NDEBUG
|
||||
} else {
|
||||
assert(T[s] == c1);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Construct the BWTed string by using
|
||||
the sorted order of type B suffixes. */
|
||||
k = SA + BUCKET_A(c2 = T[n - 1]);
|
||||
if (T[n - 2] < c2) {
|
||||
if (((n - 1) & mod) == 0) indexes[(n - 1) / (mod + 1) - 1] = k - SA;
|
||||
*k++ = ~((int)T[n - 2]);
|
||||
}
|
||||
else {
|
||||
*k++ = n - 1;
|
||||
}
|
||||
|
||||
/* Scan the suffix array from left to right. */
|
||||
for(i = SA, j = SA + n, orig = SA; i < j; ++i) {
|
||||
if(0 < (s = *i)) {
|
||||
assert(T[s - 1] >= T[s]);
|
||||
|
||||
if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = i - SA;
|
||||
|
||||
c0 = T[--s];
|
||||
*i = c0;
|
||||
if(c0 != c2) {
|
||||
BUCKET_A(c2) = k - SA;
|
||||
k = SA + BUCKET_A(c2 = c0);
|
||||
}
|
||||
assert(i < k);
|
||||
if((0 < s) && (T[s - 1] < c0)) {
|
||||
if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = k - SA;
|
||||
*k++ = ~((int)T[s - 1]);
|
||||
} else
|
||||
*k++ = s;
|
||||
} else if(s != 0) {
|
||||
*i = ~s;
|
||||
} else {
|
||||
orig = i;
|
||||
}
|
||||
}
|
||||
|
||||
return orig - SA;
|
||||
}
|
||||
|
||||
|
||||
/*---------------------------------------------------------------------------*/
|
||||
|
||||
/*- Function -*/
|
||||
|
||||
int
|
||||
divsufsort(const unsigned char *T, int *SA, int n, int openMP) {
|
||||
int *bucket_A, *bucket_B;
|
||||
int m;
|
||||
int err = 0;
|
||||
|
||||
/* Check arguments. */
|
||||
if((T == NULL) || (SA == NULL) || (n < 0)) { return -1; }
|
||||
else if(n == 0) { return 0; }
|
||||
else if(n == 1) { SA[0] = 0; return 0; }
|
||||
else if(n == 2) { m = (T[0] < T[1]); SA[m ^ 1] = 0, SA[m] = 1; return 0; }
|
||||
|
||||
bucket_A = (int *)malloc(BUCKET_A_SIZE * sizeof(int));
|
||||
bucket_B = (int *)malloc(BUCKET_B_SIZE * sizeof(int));
|
||||
|
||||
/* Suffixsort. */
|
||||
if((bucket_A != NULL) && (bucket_B != NULL)) {
|
||||
m = sort_typeBstar(T, SA, bucket_A, bucket_B, n, openMP);
|
||||
construct_SA(T, SA, bucket_A, bucket_B, n, m);
|
||||
} else {
|
||||
err = -2;
|
||||
}
|
||||
|
||||
free(bucket_B);
|
||||
free(bucket_A);
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
int
|
||||
divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * num_indexes, int * indexes, int openMP) {
|
||||
int *B;
|
||||
int *bucket_A, *bucket_B;
|
||||
int m, pidx, i;
|
||||
|
||||
/* Check arguments. */
|
||||
if((T == NULL) || (U == NULL) || (n < 0)) { return -1; }
|
||||
else if(n <= 1) { if(n == 1) { U[0] = T[0]; } return n; }
|
||||
|
||||
if((B = A) == NULL) { B = (int *)malloc((size_t)(n + 1) * sizeof(int)); }
|
||||
bucket_A = (int *)malloc(BUCKET_A_SIZE * sizeof(int));
|
||||
bucket_B = (int *)malloc(BUCKET_B_SIZE * sizeof(int));
|
||||
|
||||
/* Burrows-Wheeler Transform. */
|
||||
if((B != NULL) && (bucket_A != NULL) && (bucket_B != NULL)) {
|
||||
m = sort_typeBstar(T, B, bucket_A, bucket_B, n, openMP);
|
||||
|
||||
if (num_indexes == NULL || indexes == NULL) {
|
||||
pidx = construct_BWT(T, B, bucket_A, bucket_B, n, m);
|
||||
} else {
|
||||
pidx = construct_BWT_indexes(T, B, bucket_A, bucket_B, n, m, num_indexes, indexes);
|
||||
}
|
||||
|
||||
/* Copy to output string. */
|
||||
U[0] = T[n - 1];
|
||||
for(i = 0; i < pidx; ++i) { U[i + 1] = (unsigned char)B[i]; }
|
||||
for(i += 1; i < n; ++i) { U[i] = (unsigned char)B[i]; }
|
||||
pidx += 1;
|
||||
} else {
|
||||
pidx = -2;
|
||||
}
|
||||
|
||||
free(bucket_B);
|
||||
free(bucket_A);
|
||||
if(A == NULL) { free(B); }
|
||||
|
||||
return pidx;
|
||||
}
|
||||
|
||||
+4
-15
@@ -71,7 +71,7 @@ RUST_TARGET_32 ?= i686-unknown-linux-gnu
|
||||
RUST_STATICLIB_32 := $(RUST_TARGET_DIR)/$(RUST_TARGET_32)/release/libzstd_rs.a
|
||||
RUST_CARGO_FLAGS := --manifest-path $(RUST_MANIFEST) --release \
|
||||
--target-dir $(RUST_TARGET_DIR) --no-default-features
|
||||
RUST_CARGO_FLAGS += --features compression,decompression,dict-builder
|
||||
RUST_CARGO_FLAGS += --features compression,decompression
|
||||
ifneq ($(RUST_HUF_FEATURE),)
|
||||
RUST_CARGO_FLAGS += --features $(RUST_HUF_FEATURE)
|
||||
endif
|
||||
@@ -96,7 +96,7 @@ $(RUST_CLI_STATICLIB): $(RUST_CLI_SOURCES)
|
||||
$(RUST_CLI_STATICLIB_32): $(RUST_CLI_SOURCES)
|
||||
$(CARGO) build $(RUST_CLI_CARGO_FLAGS) --target $(RUST_TARGET_32)
|
||||
|
||||
RUST_DECOMPRESS_BUILD_CONFIG := lib-c0-d1-b0-$(RUST_BUILD_CONFIG)
|
||||
RUST_DECOMPRESS_BUILD_CONFIG := lib-c0-d1-$(RUST_BUILD_CONFIG)
|
||||
RUST_DECOMPRESS_TARGET_DIR := $(RUST_DIR)/target/$(RUST_DECOMPRESS_BUILD_CONFIG)
|
||||
RUST_DECOMPRESS_STATICLIB := $(RUST_DECOMPRESS_TARGET_DIR)/release/libzstd_rs.a
|
||||
RUST_DECOMPRESS_CARGO_FLAGS := --manifest-path $(RUST_MANIFEST) --release \
|
||||
@@ -119,7 +119,7 @@ RUST_DECOMPRESS_CLI_CARGO_FLAGS := --manifest-path $(RUST_CLI_MANIFEST) --releas
|
||||
$(RUST_DECOMPRESS_CLI_STATICLIB): $(RUST_CLI_SOURCES)
|
||||
$(CARGO) build $(RUST_DECOMPRESS_CLI_CARGO_FLAGS)
|
||||
|
||||
RUST_COMPRESS_BUILD_CONFIG := lib-c1-d0-b0-$(RUST_BUILD_CONFIG)
|
||||
RUST_COMPRESS_BUILD_CONFIG := lib-c1-d0-$(RUST_BUILD_CONFIG)
|
||||
RUST_COMPRESS_TARGET_DIR := $(RUST_DIR)/target/$(RUST_COMPRESS_BUILD_CONFIG)
|
||||
RUST_COMPRESS_STATICLIB := $(RUST_COMPRESS_TARGET_DIR)/release/libzstd_rs.a
|
||||
RUST_COMPRESS_CARGO_FLAGS := --manifest-path $(RUST_MANIFEST) --release \
|
||||
@@ -129,17 +129,6 @@ RUST_COMPRESS_CARGO_FLAGS := --manifest-path $(RUST_MANIFEST) --release \
|
||||
$(RUST_COMPRESS_STATICLIB): $(RUST_SOURCES)
|
||||
$(CARGO) build $(RUST_COMPRESS_CARGO_FLAGS)
|
||||
|
||||
RUST_DICTBUILDER_BUILD_CONFIG := lib-c1-d0-b1-$(RUST_BUILD_CONFIG)
|
||||
RUST_DICTBUILDER_TARGET_DIR := $(RUST_DIR)/target/$(RUST_DICTBUILDER_BUILD_CONFIG)
|
||||
RUST_DICTBUILDER_STATICLIB := $(RUST_DICTBUILDER_TARGET_DIR)/release/libzstd_rs.a
|
||||
RUST_DICTBUILDER_CARGO_FLAGS := --manifest-path $(RUST_MANIFEST) --release \
|
||||
--target-dir $(RUST_DICTBUILDER_TARGET_DIR) \
|
||||
--no-default-features \
|
||||
--features compression,dict-builder
|
||||
|
||||
$(RUST_DICTBUILDER_STATICLIB): $(RUST_SOURCES)
|
||||
$(CARGO) build $(RUST_DICTBUILDER_CARGO_FLAGS)
|
||||
|
||||
RUST_COMPRESS_CLI_BUILD_CONFIG := cli-c1-d0-$(RUST_BUILD_CONFIG)
|
||||
RUST_COMPRESS_CLI_TARGET_DIR := $(RUST_DIR)/target/$(RUST_COMPRESS_CLI_BUILD_CONFIG)
|
||||
RUST_COMPRESS_CLI_STATICLIB := $(RUST_COMPRESS_CLI_TARGET_DIR)/release/libzstd_cli_rs.a
|
||||
@@ -423,7 +412,7 @@ zstd-compress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) zstdcli.c util.c ti
|
||||
|
||||
## zstd-dictBuilder: executable supporting dictionary creation and compression (only)
|
||||
CLEAN += zstd-dictBuilder
|
||||
zstd-dictBuilder: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c dibio.c $(RUST_DICTBUILDER_STATICLIB) $(RUST_COMPRESS_CLI_STATICLIB)
|
||||
zstd-dictBuilder: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c dibio.c $(RUST_COMPRESS_STATICLIB) $(RUST_COMPRESS_CLI_STATICLIB)
|
||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODECOMPRESS -DZSTD_NOTRACE $^ -o $@$(EXT)
|
||||
|
||||
RUST_DIRECT_LINK_TARGETS := zstd32 zstd-nolegacy zstd-small zstd-frugal \
|
||||
|
||||
+1
-2
@@ -7,10 +7,9 @@ edition = "2021"
|
||||
crate-type = ["staticlib"]
|
||||
|
||||
[features]
|
||||
default = ["compression", "decompression", "dict-builder"]
|
||||
default = ["compression", "decompression"]
|
||||
compression = []
|
||||
decompression = []
|
||||
dict-builder = []
|
||||
huf-force-decompress-x1 = []
|
||||
huf-force-decompress-x2 = []
|
||||
|
||||
|
||||
+13
-12
@@ -31,6 +31,13 @@ zstd ABI:
|
||||
- `zstd_presplit` chooses split points for full compression blocks.
|
||||
- `zstd_compress_literals` emits raw, RLE, and Huffman literal sections
|
||||
while preserving the compressor's Huffman-table repeat state.
|
||||
- `zstd_compress_stats` converts stored sequences into symbol codes,
|
||||
selects each block's symbol encoding types, compresses a seqStore's
|
||||
literals and sequences into a compressed-block body, builds the block
|
||||
entropy statistics shared with the superblock writer and the block
|
||||
splitter, and exports collected sequences in the public `ZSTD_Sequence`
|
||||
format. Its C shims extract the sequence store, the entropy-table
|
||||
leaves, and the two `ZSTD_CCtx_params` scalars these paths read.
|
||||
- `zstd_compress_frame` serializes frame headers, skippable frames, and the
|
||||
last empty block; it takes scalar frame parameters so the C-owned
|
||||
`ZSTD_CCtx_params` layout never crosses the language boundary.
|
||||
@@ -48,10 +55,6 @@ zstd ABI:
|
||||
the dynamic-programming optimal parser itself remains in C for now.
|
||||
- `zstd_ldm` implements long-distance-match parameter selection, table
|
||||
maintenance, sequence generation, and sequence consumption.
|
||||
- Dictionary building
|
||||
- `divsufsort` constructs the suffix array that drives the legacy `ZDICT`
|
||||
trainer (`ZDICT_trainFromBuffer_legacy`). The sample analysis and
|
||||
dictionary assembly in `zdict.c`, `cover.c`, and `fastcover.c` remain C.
|
||||
- Runtime support
|
||||
- `threading` provides platform pthread wrappers required by zstd headers.
|
||||
- `pool` implements the bounded worker pool used by multithreaded compression.
|
||||
@@ -70,11 +73,10 @@ zstd ABI:
|
||||
`fileio` backend still owns file opening, safe replacement, sparse writes,
|
||||
metadata, and streaming I/O.
|
||||
|
||||
The optimal block matcher, high-level frame compression, dictionary-building
|
||||
except suffix-array construction, legacy decoding callbacks, and the CLI
|
||||
file-I/O backend are still C. They must move before the rewrite is complete.
|
||||
Keeping that boundary explicit prevents a passing hybrid build from being
|
||||
mistaken for the final all-Rust result.
|
||||
The optimal block matcher, high-level frame compression, dictionary-building,
|
||||
legacy decoding callbacks, and the CLI file-I/O backend are still C. They must
|
||||
move before the rewrite is complete. Keeping that boundary explicit prevents a
|
||||
passing hybrid build from being mistaken for the final all-Rust result.
|
||||
|
||||
## Compatibility boundary
|
||||
|
||||
@@ -85,9 +87,8 @@ makefile source list as a small shim so header configuration and platform
|
||||
preprocessor behavior stay available during the transition.
|
||||
|
||||
The library, test, and program makefiles select an archive directory for the
|
||||
active C configuration: enabled compression/decompression/dictionary-builder
|
||||
modules, default or forced HUF X1/X2, and the matching Rust target for 32-bit
|
||||
C binaries. The
|
||||
active C configuration: enabled compression/decompression modules, default or
|
||||
forced HUF X1/X2, and the matching Rust target for 32-bit C binaries. The
|
||||
native static archive flattens Rust object members rather than nesting a Rust
|
||||
archive, while the native shared library retains all migrated Rust exports.
|
||||
When the HUF mode changes, the test and program paths also rebuild cached C
|
||||
|
||||
@@ -1,2756 +0,0 @@
|
||||
#![allow(clippy::missing_safety_doc)]
|
||||
#![allow(clippy::too_many_arguments)]
|
||||
|
||||
//! Suffix-array construction for the dictionary builder.
|
||||
//!
|
||||
//! Port of `lib/dictBuilder/divsufsort.c` (libdivsufsort-lite, Copyright (c)
|
||||
//! 2003-2008 Yuta Mori, MIT license) in the exact configuration zstd compiles
|
||||
//! it with: `ALPHABET_SIZE = 256`, `SS_INSERTIONSORT_THRESHOLD = 8`,
|
||||
//! `SS_BLOCKSIZE = 1024`, and no OpenMP. Only `divsufsort()` is exported;
|
||||
//! `divbwt()` has no callers anywhere in zstd and was not ported.
|
||||
//!
|
||||
//! The C implementation walks raw `int*` cursors through the caller's SA
|
||||
//! buffer, including transient one-before-the-range positions. Every such
|
||||
//! cursor is translated to an `isize` index into one `&mut [i32]` slice
|
||||
//! covering the whole buffer, so all arithmetic — including the
|
||||
//! bitwise-complement rank marking and the C `int` value semantics — matches
|
||||
//! the original exactly while staying bounds-checked.
|
||||
|
||||
use std::os::raw::c_int;
|
||||
use std::slice;
|
||||
|
||||
const BUCKET_A_SIZE: usize = 256; /* ALPHABET_SIZE */
|
||||
const BUCKET_B_SIZE: usize = 256 * 256; /* ALPHABET_SIZE * ALPHABET_SIZE */
|
||||
const ALPHABET_SIZE: i32 = 256;
|
||||
const SS_INSERTIONSORT_THRESHOLD: isize = 8;
|
||||
const SS_BLOCKSIZE: isize = 1024;
|
||||
/* minstacksize = log(SS_BLOCKSIZE) / log(3) * 2 */
|
||||
const SS_MISORT_STACKSIZE: usize = 16;
|
||||
const SS_SMERGE_STACKSIZE: usize = 32;
|
||||
const TR_INSERTIONSORT_THRESHOLD: isize = 8;
|
||||
const TR_STACKSIZE: usize = 64;
|
||||
|
||||
#[rustfmt::skip]
|
||||
static LG_TABLE: [i32; 256] = [
|
||||
-1,0,1,1,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,
|
||||
5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,
|
||||
6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
|
||||
6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
|
||||
7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
|
||||
7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
|
||||
7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
|
||||
7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
|
||||
];
|
||||
|
||||
#[rustfmt::skip]
|
||||
static SQQ_TABLE: [i32; 256] = [
|
||||
0, 16, 22, 27, 32, 35, 39, 42, 45, 48, 50, 53, 55, 57, 59, 61,
|
||||
64, 65, 67, 69, 71, 73, 75, 76, 78, 80, 81, 83, 84, 86, 87, 89,
|
||||
90, 91, 93, 94, 96, 97, 98, 99, 101, 102, 103, 104, 106, 107, 108, 109,
|
||||
110, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126,
|
||||
128, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142,
|
||||
143, 144, 144, 145, 146, 147, 148, 149, 150, 150, 151, 152, 153, 154, 155, 155,
|
||||
156, 157, 158, 159, 160, 160, 161, 162, 163, 163, 164, 165, 166, 167, 167, 168,
|
||||
169, 170, 170, 171, 172, 173, 173, 174, 175, 176, 176, 177, 178, 178, 179, 180,
|
||||
181, 181, 182, 183, 183, 184, 185, 185, 186, 187, 187, 188, 189, 189, 190, 191,
|
||||
192, 192, 193, 193, 194, 195, 195, 196, 197, 197, 198, 199, 199, 200, 201, 201,
|
||||
202, 203, 203, 204, 204, 205, 206, 206, 207, 208, 208, 209, 209, 210, 211, 211,
|
||||
212, 212, 213, 214, 214, 215, 215, 216, 217, 217, 218, 218, 219, 219, 220, 221,
|
||||
221, 222, 222, 223, 224, 224, 225, 225, 226, 226, 227, 227, 228, 229, 229, 230,
|
||||
230, 231, 231, 232, 232, 233, 234, 234, 235, 235, 236, 236, 237, 237, 238, 238,
|
||||
239, 240, 240, 241, 241, 242, 242, 243, 243, 244, 244, 245, 245, 246, 246, 247,
|
||||
247, 248, 248, 249, 249, 250, 250, 251, 251, 252, 252, 253, 253, 254, 254, 255,
|
||||
];
|
||||
|
||||
/* `ss_ilg` in its `256 <= SS_BLOCKSIZE` configuration. */
|
||||
#[inline]
|
||||
fn ss_ilg(n: isize) -> i32 {
|
||||
let n = n as i32;
|
||||
if n & 0xff00 != 0 {
|
||||
8 + LG_TABLE[((n >> 8) & 0xff) as usize]
|
||||
} else {
|
||||
LG_TABLE[(n & 0xff) as usize]
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn ss_isqrt(x: isize) -> isize {
|
||||
if x >= SS_BLOCKSIZE * SS_BLOCKSIZE {
|
||||
return SS_BLOCKSIZE;
|
||||
}
|
||||
let x = x as i32;
|
||||
let e = if (x as u32) & 0xffff_0000 != 0 {
|
||||
if (x as u32) & 0xff00_0000 != 0 {
|
||||
24 + LG_TABLE[((x >> 24) & 0xff) as usize]
|
||||
} else {
|
||||
16 + LG_TABLE[((x >> 16) & 0xff) as usize]
|
||||
}
|
||||
} else if x & 0xff00 != 0 {
|
||||
8 + LG_TABLE[((x >> 8) & 0xff) as usize]
|
||||
} else {
|
||||
LG_TABLE[(x & 0xff) as usize]
|
||||
};
|
||||
|
||||
let mut y;
|
||||
if e >= 16 {
|
||||
y = SQQ_TABLE[(x >> ((e - 6) - (e & 1))) as usize] << ((e >> 1) - 7);
|
||||
if e >= 24 {
|
||||
y = (y + 1 + x / y) >> 1;
|
||||
}
|
||||
y = (y + 1 + x / y) >> 1;
|
||||
} else if e >= 8 {
|
||||
y = (SQQ_TABLE[(x >> ((e - 6) - (e & 1))) as usize] >> (7 - (e >> 1))) + 1;
|
||||
} else {
|
||||
return (SQQ_TABLE[x as usize] >> 4) as isize;
|
||||
}
|
||||
|
||||
(if x < y * y { y - 1 } else { y }) as isize
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------------- */
|
||||
|
||||
/// Compares two suffixes. `(p10, p11)` and `(p20, p21)` are the `p[0]`/`p[1]`
|
||||
/// pairs the C routine reads through its `const int*` arguments; passing the
|
||||
/// values directly also serves `sssort()`'s local two-element `PAi` array.
|
||||
#[inline]
|
||||
fn ss_compare(t: &[u8], p10: i32, p11: i32, p20: i32, p21: i32, depth: i32) -> i32 {
|
||||
let mut u1 = (depth + p10) as isize;
|
||||
let mut u2 = (depth + p20) as isize;
|
||||
let u1n = (p11 + 2) as isize;
|
||||
let u2n = (p21 + 2) as isize;
|
||||
|
||||
while u1 < u1n && u2 < u2n && t[u1 as usize] == t[u2 as usize] {
|
||||
u1 += 1;
|
||||
u2 += 1;
|
||||
}
|
||||
|
||||
if u1 < u1n {
|
||||
if u2 < u2n {
|
||||
t[u1 as usize] as i32 - t[u2 as usize] as i32
|
||||
} else {
|
||||
1
|
||||
}
|
||||
} else if u2 < u2n {
|
||||
-1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
|
||||
/// `ss_compare(T, p1, p2, depth)` for pointers `p1`/`p2` into the SA buffer.
|
||||
#[inline]
|
||||
fn ss_compare_pa(t: &[u8], sa: &[i32], p1: isize, p2: isize, depth: i32) -> i32 {
|
||||
ss_compare(
|
||||
t,
|
||||
sa[p1 as usize],
|
||||
sa[(p1 + 1) as usize],
|
||||
sa[p2 as usize],
|
||||
sa[(p2 + 1) as usize],
|
||||
depth,
|
||||
)
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------------- */
|
||||
|
||||
/* Insertionsort for small size groups */
|
||||
fn ss_insertionsort(t: &[u8], sa: &mut [i32], pa: isize, first: isize, last: isize, depth: i32) {
|
||||
let mut i = last - 2;
|
||||
while first <= i {
|
||||
let t0 = sa[i as usize];
|
||||
let mut j = i + 1;
|
||||
let mut r;
|
||||
loop {
|
||||
r = ss_compare_pa(t, sa, pa + t0 as isize, pa + sa[j as usize] as isize, depth);
|
||||
if r <= 0 {
|
||||
break;
|
||||
}
|
||||
loop {
|
||||
sa[(j - 1) as usize] = sa[j as usize];
|
||||
j += 1;
|
||||
if !(j < last && sa[j as usize] < 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if last <= j {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if r == 0 {
|
||||
sa[j as usize] = !sa[j as usize];
|
||||
}
|
||||
sa[(j - 1) as usize] = t0;
|
||||
i -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------------- */
|
||||
|
||||
/// `Td[PA[SA[p]]]` — the depth-`td` sorting key of the suffix stored at `p`.
|
||||
#[inline(always)]
|
||||
fn ss_key(t: &[u8], sa: &[i32], td: isize, pa: isize, p: isize) -> i32 {
|
||||
t[(td + sa[(pa + sa[p as usize] as isize) as usize] as isize) as usize] as i32
|
||||
}
|
||||
|
||||
/// `Td[v]` for an already-loaded SA element `v` (`Td[PA[v]]` in C).
|
||||
#[inline(always)]
|
||||
fn ss_key_of(t: &[u8], sa: &[i32], td: isize, pa: isize, v: i32) -> i32 {
|
||||
t[(td + sa[(pa + v as isize) as usize] as isize) as usize] as i32
|
||||
}
|
||||
|
||||
/// `Td[PA[SA[p]] - 1]` — the character preceding the depth-`td` key.
|
||||
#[inline(always)]
|
||||
fn ss_key_pred(t: &[u8], sa: &[i32], td: isize, pa: isize, p: isize) -> i32 {
|
||||
t[(td + sa[(pa + sa[p as usize] as isize) as usize] as isize - 1) as usize] as i32
|
||||
}
|
||||
|
||||
fn ss_fixdown(t: &[u8], td: isize, sa: &mut [i32], pa: isize, base: isize, i: isize, size: isize) {
|
||||
let mut i = i;
|
||||
let v = sa[(base + i) as usize];
|
||||
let c = ss_key_of(t, sa, td, pa, v);
|
||||
loop {
|
||||
let mut j = 2 * i + 1;
|
||||
if j >= size {
|
||||
break;
|
||||
}
|
||||
let mut k = j;
|
||||
j += 1;
|
||||
let mut d = ss_key(t, sa, td, pa, base + k);
|
||||
let e = ss_key(t, sa, td, pa, base + j);
|
||||
if d < e {
|
||||
k = j;
|
||||
d = e;
|
||||
}
|
||||
if d <= c {
|
||||
break;
|
||||
}
|
||||
sa[(base + i) as usize] = sa[(base + k) as usize];
|
||||
i = k;
|
||||
}
|
||||
sa[(base + i) as usize] = v;
|
||||
}
|
||||
|
||||
/* Simple top-down heapsort. */
|
||||
fn ss_heapsort(t: &[u8], td: isize, sa: &mut [i32], pa: isize, base: isize, size: isize) {
|
||||
let mut m = size;
|
||||
if size % 2 == 0 {
|
||||
m -= 1;
|
||||
if ss_key(t, sa, td, pa, base + m / 2) < ss_key(t, sa, td, pa, base + m) {
|
||||
sa.swap((base + m) as usize, (base + m / 2) as usize);
|
||||
}
|
||||
}
|
||||
|
||||
let mut i = m / 2 - 1;
|
||||
while 0 <= i {
|
||||
ss_fixdown(t, td, sa, pa, base, i, m);
|
||||
i -= 1;
|
||||
}
|
||||
if size % 2 == 0 {
|
||||
sa.swap(base as usize, (base + m) as usize);
|
||||
ss_fixdown(t, td, sa, pa, base, 0, m);
|
||||
}
|
||||
let mut i = m - 1;
|
||||
while 0 < i {
|
||||
let t0 = sa[base as usize];
|
||||
sa[base as usize] = sa[(base + i) as usize];
|
||||
ss_fixdown(t, td, sa, pa, base, 0, i);
|
||||
sa[(base + i) as usize] = t0;
|
||||
i -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------------- */
|
||||
|
||||
/* Returns the median of three elements. */
|
||||
#[inline]
|
||||
fn ss_median3(
|
||||
t: &[u8],
|
||||
sa: &[i32],
|
||||
td: isize,
|
||||
pa: isize,
|
||||
v1: isize,
|
||||
v2: isize,
|
||||
v3: isize,
|
||||
) -> isize {
|
||||
let mut v1 = v1;
|
||||
let mut v2 = v2;
|
||||
if ss_key(t, sa, td, pa, v1) > ss_key(t, sa, td, pa, v2) {
|
||||
std::mem::swap(&mut v1, &mut v2);
|
||||
}
|
||||
if ss_key(t, sa, td, pa, v2) > ss_key(t, sa, td, pa, v3) {
|
||||
if ss_key(t, sa, td, pa, v1) > ss_key(t, sa, td, pa, v3) {
|
||||
return v1;
|
||||
}
|
||||
return v3;
|
||||
}
|
||||
v2
|
||||
}
|
||||
|
||||
/* Returns the median of five elements. */
|
||||
#[inline]
|
||||
fn ss_median5(
|
||||
t: &[u8],
|
||||
sa: &[i32],
|
||||
td: isize,
|
||||
pa: isize,
|
||||
v1: isize,
|
||||
v2: isize,
|
||||
v3: isize,
|
||||
v4: isize,
|
||||
v5: isize,
|
||||
) -> isize {
|
||||
let mut v1 = v1;
|
||||
let mut v2 = v2;
|
||||
let mut v3 = v3;
|
||||
let mut v4 = v4;
|
||||
let mut v5 = v5;
|
||||
if ss_key(t, sa, td, pa, v2) > ss_key(t, sa, td, pa, v3) {
|
||||
std::mem::swap(&mut v2, &mut v3);
|
||||
}
|
||||
if ss_key(t, sa, td, pa, v4) > ss_key(t, sa, td, pa, v5) {
|
||||
std::mem::swap(&mut v4, &mut v5);
|
||||
}
|
||||
if ss_key(t, sa, td, pa, v2) > ss_key(t, sa, td, pa, v4) {
|
||||
std::mem::swap(&mut v2, &mut v4);
|
||||
std::mem::swap(&mut v3, &mut v5);
|
||||
}
|
||||
if ss_key(t, sa, td, pa, v1) > ss_key(t, sa, td, pa, v3) {
|
||||
std::mem::swap(&mut v1, &mut v3);
|
||||
}
|
||||
if ss_key(t, sa, td, pa, v1) > ss_key(t, sa, td, pa, v4) {
|
||||
std::mem::swap(&mut v1, &mut v4);
|
||||
std::mem::swap(&mut v3, &mut v5);
|
||||
}
|
||||
if ss_key(t, sa, td, pa, v3) > ss_key(t, sa, td, pa, v4) {
|
||||
return v4;
|
||||
}
|
||||
v3
|
||||
}
|
||||
|
||||
/* Returns the pivot element. */
|
||||
#[inline]
|
||||
fn ss_pivot(t: &[u8], sa: &[i32], td: isize, pa: isize, first: isize, last: isize) -> isize {
|
||||
let mut t0 = last - first;
|
||||
let middle = first + t0 / 2;
|
||||
|
||||
if t0 <= 512 {
|
||||
if t0 <= 32 {
|
||||
return ss_median3(t, sa, td, pa, first, middle, last - 1);
|
||||
}
|
||||
t0 >>= 2;
|
||||
return ss_median5(
|
||||
t,
|
||||
sa,
|
||||
td,
|
||||
pa,
|
||||
first,
|
||||
first + t0,
|
||||
middle,
|
||||
last - 1 - t0,
|
||||
last - 1,
|
||||
);
|
||||
}
|
||||
t0 >>= 3;
|
||||
let first = ss_median3(t, sa, td, pa, first, first + t0, first + (t0 << 1));
|
||||
let middle = ss_median3(t, sa, td, pa, middle - t0, middle, middle + t0);
|
||||
let last = ss_median3(t, sa, td, pa, last - 1 - (t0 << 1), last - 1 - t0, last - 1);
|
||||
ss_median3(t, sa, td, pa, first, middle, last)
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------------- */
|
||||
|
||||
/* Binary partition for substrings. */
|
||||
/* The `>= x + 1` comparison deliberately mirrors the C expression shape. */
|
||||
#[allow(clippy::int_plus_one)]
|
||||
fn ss_partition(sa: &mut [i32], pa: isize, first: isize, last: isize, depth: i32) -> isize {
|
||||
let mut a = first - 1;
|
||||
let mut b = last;
|
||||
loop {
|
||||
loop {
|
||||
a += 1;
|
||||
if !(a < b) {
|
||||
break;
|
||||
}
|
||||
if !(sa[(pa + sa[a as usize] as isize) as usize] + depth
|
||||
>= sa[(pa + sa[a as usize] as isize + 1) as usize] + 1)
|
||||
{
|
||||
break;
|
||||
}
|
||||
sa[a as usize] = !sa[a as usize];
|
||||
}
|
||||
loop {
|
||||
b -= 1;
|
||||
if !(a < b) {
|
||||
break;
|
||||
}
|
||||
if !(sa[(pa + sa[b as usize] as isize) as usize] + depth
|
||||
< sa[(pa + sa[b as usize] as isize + 1) as usize] + 1)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
if b <= a {
|
||||
break;
|
||||
}
|
||||
let t0 = !sa[b as usize];
|
||||
sa[b as usize] = sa[a as usize];
|
||||
sa[a as usize] = t0;
|
||||
}
|
||||
if first < a {
|
||||
sa[first as usize] = !sa[first as usize];
|
||||
}
|
||||
a
|
||||
}
|
||||
|
||||
/* Multikey introsort for medium size groups. */
|
||||
fn ss_mintrosort(t: &[u8], sa: &mut [i32], pa: isize, first: isize, last: isize, depth: i32) {
|
||||
let mut stack = [(0isize, 0isize, 0i32, 0i32); SS_MISORT_STACKSIZE];
|
||||
let mut ssize = 0usize;
|
||||
let mut first = first;
|
||||
let mut last = last;
|
||||
let mut depth = depth;
|
||||
let mut limit = ss_ilg(last - first);
|
||||
let mut x: i32 = 0;
|
||||
|
||||
loop {
|
||||
if last - first <= SS_INSERTIONSORT_THRESHOLD {
|
||||
if 1 < last - first {
|
||||
ss_insertionsort(t, sa, pa, first, last, depth);
|
||||
}
|
||||
/* STACK_POP */
|
||||
if ssize == 0 {
|
||||
return;
|
||||
}
|
||||
ssize -= 1;
|
||||
(first, last, depth, limit) = stack[ssize];
|
||||
continue;
|
||||
}
|
||||
|
||||
let td = depth as isize;
|
||||
if limit == 0 {
|
||||
ss_heapsort(t, td, sa, pa, first, last - first);
|
||||
}
|
||||
limit -= 1;
|
||||
if limit < 0 {
|
||||
let mut a = first + 1;
|
||||
let mut v = ss_key(t, sa, td, pa, first);
|
||||
while a < last {
|
||||
x = ss_key(t, sa, td, pa, a);
|
||||
if x != v {
|
||||
if 1 < a - first {
|
||||
break;
|
||||
}
|
||||
v = x;
|
||||
first = a;
|
||||
}
|
||||
a += 1;
|
||||
}
|
||||
if ss_key_pred(t, sa, td, pa, first) < v {
|
||||
first = ss_partition(sa, pa, first, a, depth);
|
||||
}
|
||||
if a - first <= last - a {
|
||||
if 1 < a - first {
|
||||
stack[ssize] = (a, last, depth, -1);
|
||||
ssize += 1;
|
||||
last = a;
|
||||
depth += 1;
|
||||
limit = ss_ilg(a - first);
|
||||
} else {
|
||||
first = a;
|
||||
limit = -1;
|
||||
}
|
||||
} else if 1 < last - a {
|
||||
stack[ssize] = (first, a, depth + 1, ss_ilg(a - first));
|
||||
ssize += 1;
|
||||
first = a;
|
||||
limit = -1;
|
||||
} else {
|
||||
last = a;
|
||||
depth += 1;
|
||||
limit = ss_ilg(a - first);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
/* choose pivot */
|
||||
let mut a = ss_pivot(t, sa, td, pa, first, last);
|
||||
let v = ss_key(t, sa, td, pa, a);
|
||||
sa.swap(first as usize, a as usize);
|
||||
|
||||
/* partition */
|
||||
let mut b = first;
|
||||
loop {
|
||||
b += 1;
|
||||
if !(b < last) {
|
||||
break;
|
||||
}
|
||||
x = ss_key(t, sa, td, pa, b);
|
||||
if x != v {
|
||||
break;
|
||||
}
|
||||
}
|
||||
a = b;
|
||||
if a < last && x < v {
|
||||
loop {
|
||||
b += 1;
|
||||
if !(b < last) {
|
||||
break;
|
||||
}
|
||||
x = ss_key(t, sa, td, pa, b);
|
||||
if !(x <= v) {
|
||||
break;
|
||||
}
|
||||
if x == v {
|
||||
sa.swap(b as usize, a as usize);
|
||||
a += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut c = last;
|
||||
loop {
|
||||
c -= 1;
|
||||
if !(b < c) {
|
||||
break;
|
||||
}
|
||||
x = ss_key(t, sa, td, pa, c);
|
||||
if x != v {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let mut d = c;
|
||||
if b < d && x > v {
|
||||
loop {
|
||||
c -= 1;
|
||||
if !(b < c) {
|
||||
break;
|
||||
}
|
||||
x = ss_key(t, sa, td, pa, c);
|
||||
if !(x >= v) {
|
||||
break;
|
||||
}
|
||||
if x == v {
|
||||
sa.swap(c as usize, d as usize);
|
||||
d -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
while b < c {
|
||||
sa.swap(b as usize, c as usize);
|
||||
loop {
|
||||
b += 1;
|
||||
if !(b < c) {
|
||||
break;
|
||||
}
|
||||
x = ss_key(t, sa, td, pa, b);
|
||||
if !(x <= v) {
|
||||
break;
|
||||
}
|
||||
if x == v {
|
||||
sa.swap(b as usize, a as usize);
|
||||
a += 1;
|
||||
}
|
||||
}
|
||||
loop {
|
||||
c -= 1;
|
||||
if !(b < c) {
|
||||
break;
|
||||
}
|
||||
x = ss_key(t, sa, td, pa, c);
|
||||
if !(x >= v) {
|
||||
break;
|
||||
}
|
||||
if x == v {
|
||||
sa.swap(c as usize, d as usize);
|
||||
d -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if a <= d {
|
||||
c = b - 1;
|
||||
|
||||
let mut s = a - first;
|
||||
let t0 = b - a;
|
||||
if s > t0 {
|
||||
s = t0;
|
||||
}
|
||||
let mut e = first;
|
||||
let mut f = b - s;
|
||||
while 0 < s {
|
||||
sa.swap(e as usize, f as usize);
|
||||
s -= 1;
|
||||
e += 1;
|
||||
f += 1;
|
||||
}
|
||||
let mut s = d - c;
|
||||
let t0 = last - d - 1;
|
||||
if s > t0 {
|
||||
s = t0;
|
||||
}
|
||||
let mut e = b;
|
||||
let mut f = last - s;
|
||||
while 0 < s {
|
||||
sa.swap(e as usize, f as usize);
|
||||
s -= 1;
|
||||
e += 1;
|
||||
f += 1;
|
||||
}
|
||||
|
||||
a = first + (b - a);
|
||||
c = last - (d - c);
|
||||
b = if v <= ss_key_pred(t, sa, td, pa, a) {
|
||||
a
|
||||
} else {
|
||||
ss_partition(sa, pa, a, c, depth)
|
||||
};
|
||||
|
||||
if a - first <= last - c {
|
||||
if last - c <= c - b {
|
||||
stack[ssize] = (b, c, depth + 1, ss_ilg(c - b));
|
||||
ssize += 1;
|
||||
stack[ssize] = (c, last, depth, limit);
|
||||
ssize += 1;
|
||||
last = a;
|
||||
} else if a - first <= c - b {
|
||||
stack[ssize] = (c, last, depth, limit);
|
||||
ssize += 1;
|
||||
stack[ssize] = (b, c, depth + 1, ss_ilg(c - b));
|
||||
ssize += 1;
|
||||
last = a;
|
||||
} else {
|
||||
stack[ssize] = (c, last, depth, limit);
|
||||
ssize += 1;
|
||||
stack[ssize] = (first, a, depth, limit);
|
||||
ssize += 1;
|
||||
first = b;
|
||||
last = c;
|
||||
depth += 1;
|
||||
limit = ss_ilg(c - b);
|
||||
}
|
||||
} else if a - first <= c - b {
|
||||
stack[ssize] = (b, c, depth + 1, ss_ilg(c - b));
|
||||
ssize += 1;
|
||||
stack[ssize] = (first, a, depth, limit);
|
||||
ssize += 1;
|
||||
first = c;
|
||||
} else if last - c <= c - b {
|
||||
stack[ssize] = (first, a, depth, limit);
|
||||
ssize += 1;
|
||||
stack[ssize] = (b, c, depth + 1, ss_ilg(c - b));
|
||||
ssize += 1;
|
||||
first = c;
|
||||
} else {
|
||||
stack[ssize] = (first, a, depth, limit);
|
||||
ssize += 1;
|
||||
stack[ssize] = (c, last, depth, limit);
|
||||
ssize += 1;
|
||||
first = b;
|
||||
last = c;
|
||||
depth += 1;
|
||||
limit = ss_ilg(c - b);
|
||||
}
|
||||
} else {
|
||||
limit += 1;
|
||||
if ss_key_pred(t, sa, td, pa, first) < v {
|
||||
first = ss_partition(sa, pa, first, last, depth);
|
||||
limit = ss_ilg(last - first);
|
||||
}
|
||||
depth += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------------- */
|
||||
|
||||
#[inline]
|
||||
fn ss_blockswap(sa: &mut [i32], a: isize, b: isize, n: isize) {
|
||||
let mut a = a;
|
||||
let mut b = b;
|
||||
let mut n = n;
|
||||
while 0 < n {
|
||||
sa.swap(a as usize, b as usize);
|
||||
n -= 1;
|
||||
a += 1;
|
||||
b += 1;
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn ss_rotate(sa: &mut [i32], first: isize, middle: isize, last: isize) {
|
||||
let mut first = first;
|
||||
let mut last = last;
|
||||
let mut l = middle - first;
|
||||
let mut r = last - middle;
|
||||
while 0 < l && 0 < r {
|
||||
if l == r {
|
||||
ss_blockswap(sa, first, middle, l);
|
||||
break;
|
||||
}
|
||||
if l < r {
|
||||
let mut a = last - 1;
|
||||
let mut b = middle - 1;
|
||||
let mut t0 = sa[a as usize];
|
||||
loop {
|
||||
sa[a as usize] = sa[b as usize];
|
||||
a -= 1;
|
||||
sa[b as usize] = sa[a as usize];
|
||||
b -= 1;
|
||||
if b < first {
|
||||
sa[a as usize] = t0;
|
||||
last = a;
|
||||
r -= l + 1;
|
||||
if r <= l {
|
||||
break;
|
||||
}
|
||||
a -= 1;
|
||||
b = middle - 1;
|
||||
t0 = sa[a as usize];
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let mut a = first;
|
||||
let mut b = middle;
|
||||
let mut t0 = sa[a as usize];
|
||||
loop {
|
||||
sa[a as usize] = sa[b as usize];
|
||||
a += 1;
|
||||
sa[b as usize] = sa[a as usize];
|
||||
b += 1;
|
||||
if last <= b {
|
||||
sa[a as usize] = t0;
|
||||
first = a + 1;
|
||||
l -= r + 1;
|
||||
if l <= r {
|
||||
break;
|
||||
}
|
||||
a += 1;
|
||||
b = middle;
|
||||
t0 = sa[a as usize];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------------- */
|
||||
|
||||
fn ss_inplacemerge(
|
||||
t: &[u8],
|
||||
sa: &mut [i32],
|
||||
pa: isize,
|
||||
first: isize,
|
||||
middle: isize,
|
||||
last: isize,
|
||||
depth: i32,
|
||||
) {
|
||||
let mut middle = middle;
|
||||
let mut last = last;
|
||||
loop {
|
||||
let x: i32;
|
||||
let p: isize;
|
||||
if sa[(last - 1) as usize] < 0 {
|
||||
x = 1;
|
||||
p = pa + (!sa[(last - 1) as usize]) as isize;
|
||||
} else {
|
||||
x = 0;
|
||||
p = pa + sa[(last - 1) as usize] as isize;
|
||||
}
|
||||
let mut a = first;
|
||||
let mut len = middle - first;
|
||||
let mut half = len >> 1;
|
||||
let mut r: i32 = -1;
|
||||
while 0 < len {
|
||||
let b = a + half;
|
||||
let bv = sa[b as usize];
|
||||
let q = ss_compare_pa(
|
||||
t,
|
||||
sa,
|
||||
pa + (if 0 <= bv { bv } else { !bv }) as isize,
|
||||
p,
|
||||
depth,
|
||||
);
|
||||
if q < 0 {
|
||||
a = b + 1;
|
||||
half -= (len & 1) ^ 1;
|
||||
} else {
|
||||
r = q;
|
||||
}
|
||||
len = half;
|
||||
half >>= 1;
|
||||
}
|
||||
if a < middle {
|
||||
if r == 0 {
|
||||
sa[a as usize] = !sa[a as usize];
|
||||
}
|
||||
ss_rotate(sa, a, middle, last);
|
||||
last -= middle - a;
|
||||
middle = a;
|
||||
if first == middle {
|
||||
break;
|
||||
}
|
||||
}
|
||||
last -= 1;
|
||||
if x != 0 {
|
||||
loop {
|
||||
last -= 1;
|
||||
if !(sa[last as usize] < 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if middle == last {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------------- */
|
||||
|
||||
/* Merge-forward with internal buffer. */
|
||||
fn ss_mergeforward(
|
||||
t: &[u8],
|
||||
sa: &mut [i32],
|
||||
pa: isize,
|
||||
first: isize,
|
||||
middle: isize,
|
||||
last: isize,
|
||||
buf: isize,
|
||||
depth: i32,
|
||||
) {
|
||||
let bufend = buf + (middle - first) - 1;
|
||||
ss_blockswap(sa, buf, first, middle - first);
|
||||
|
||||
let mut a = first;
|
||||
let t0 = sa[a as usize];
|
||||
let mut b = buf;
|
||||
let mut c = middle;
|
||||
loop {
|
||||
let r = ss_compare_pa(
|
||||
t,
|
||||
sa,
|
||||
pa + sa[b as usize] as isize,
|
||||
pa + sa[c as usize] as isize,
|
||||
depth,
|
||||
);
|
||||
if r < 0 {
|
||||
loop {
|
||||
sa[a as usize] = sa[b as usize];
|
||||
a += 1;
|
||||
if bufend <= b {
|
||||
sa[bufend as usize] = t0;
|
||||
return;
|
||||
}
|
||||
sa[b as usize] = sa[a as usize];
|
||||
b += 1;
|
||||
if !(sa[b as usize] < 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else if r > 0 {
|
||||
loop {
|
||||
sa[a as usize] = sa[c as usize];
|
||||
a += 1;
|
||||
sa[c as usize] = sa[a as usize];
|
||||
c += 1;
|
||||
if last <= c {
|
||||
while b < bufend {
|
||||
sa[a as usize] = sa[b as usize];
|
||||
a += 1;
|
||||
sa[b as usize] = sa[a as usize];
|
||||
b += 1;
|
||||
}
|
||||
sa[a as usize] = sa[b as usize];
|
||||
sa[b as usize] = t0;
|
||||
return;
|
||||
}
|
||||
if !(sa[c as usize] < 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
sa[c as usize] = !sa[c as usize];
|
||||
loop {
|
||||
sa[a as usize] = sa[b as usize];
|
||||
a += 1;
|
||||
if bufend <= b {
|
||||
sa[bufend as usize] = t0;
|
||||
return;
|
||||
}
|
||||
sa[b as usize] = sa[a as usize];
|
||||
b += 1;
|
||||
if !(sa[b as usize] < 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
loop {
|
||||
sa[a as usize] = sa[c as usize];
|
||||
a += 1;
|
||||
sa[c as usize] = sa[a as usize];
|
||||
c += 1;
|
||||
if last <= c {
|
||||
while b < bufend {
|
||||
sa[a as usize] = sa[b as usize];
|
||||
a += 1;
|
||||
sa[b as usize] = sa[a as usize];
|
||||
b += 1;
|
||||
}
|
||||
sa[a as usize] = sa[b as usize];
|
||||
sa[b as usize] = t0;
|
||||
return;
|
||||
}
|
||||
if !(sa[c as usize] < 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Merge-backward with internal buffer. */
|
||||
fn ss_mergebackward(
|
||||
t: &[u8],
|
||||
sa: &mut [i32],
|
||||
pa: isize,
|
||||
first: isize,
|
||||
middle: isize,
|
||||
last: isize,
|
||||
buf: isize,
|
||||
depth: i32,
|
||||
) {
|
||||
let bufend = buf + (last - middle) - 1;
|
||||
ss_blockswap(sa, buf, middle, last - middle);
|
||||
|
||||
let mut x = 0i32;
|
||||
let mut p1: isize;
|
||||
let mut p2: isize;
|
||||
if sa[bufend as usize] < 0 {
|
||||
p1 = pa + (!sa[bufend as usize]) as isize;
|
||||
x |= 1;
|
||||
} else {
|
||||
p1 = pa + sa[bufend as usize] as isize;
|
||||
}
|
||||
if sa[(middle - 1) as usize] < 0 {
|
||||
p2 = pa + (!sa[(middle - 1) as usize]) as isize;
|
||||
x |= 2;
|
||||
} else {
|
||||
p2 = pa + sa[(middle - 1) as usize] as isize;
|
||||
}
|
||||
let mut a = last - 1;
|
||||
let t0 = sa[a as usize];
|
||||
let mut b = bufend;
|
||||
let mut c = middle - 1;
|
||||
loop {
|
||||
let r = ss_compare_pa(t, sa, p1, p2, depth);
|
||||
if 0 < r {
|
||||
if x & 1 != 0 {
|
||||
loop {
|
||||
sa[a as usize] = sa[b as usize];
|
||||
a -= 1;
|
||||
sa[b as usize] = sa[a as usize];
|
||||
b -= 1;
|
||||
if !(sa[b as usize] < 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
x ^= 1;
|
||||
}
|
||||
sa[a as usize] = sa[b as usize];
|
||||
a -= 1;
|
||||
if b <= buf {
|
||||
sa[buf as usize] = t0;
|
||||
break;
|
||||
}
|
||||
sa[b as usize] = sa[a as usize];
|
||||
b -= 1;
|
||||
if sa[b as usize] < 0 {
|
||||
p1 = pa + (!sa[b as usize]) as isize;
|
||||
x |= 1;
|
||||
} else {
|
||||
p1 = pa + sa[b as usize] as isize;
|
||||
}
|
||||
} else if r < 0 {
|
||||
if x & 2 != 0 {
|
||||
loop {
|
||||
sa[a as usize] = sa[c as usize];
|
||||
a -= 1;
|
||||
sa[c as usize] = sa[a as usize];
|
||||
c -= 1;
|
||||
if !(sa[c as usize] < 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
x ^= 2;
|
||||
}
|
||||
sa[a as usize] = sa[c as usize];
|
||||
a -= 1;
|
||||
sa[c as usize] = sa[a as usize];
|
||||
c -= 1;
|
||||
if c < first {
|
||||
while buf < b {
|
||||
sa[a as usize] = sa[b as usize];
|
||||
a -= 1;
|
||||
sa[b as usize] = sa[a as usize];
|
||||
b -= 1;
|
||||
}
|
||||
sa[a as usize] = sa[b as usize];
|
||||
sa[b as usize] = t0;
|
||||
break;
|
||||
}
|
||||
if sa[c as usize] < 0 {
|
||||
p2 = pa + (!sa[c as usize]) as isize;
|
||||
x |= 2;
|
||||
} else {
|
||||
p2 = pa + sa[c as usize] as isize;
|
||||
}
|
||||
} else {
|
||||
if x & 1 != 0 {
|
||||
loop {
|
||||
sa[a as usize] = sa[b as usize];
|
||||
a -= 1;
|
||||
sa[b as usize] = sa[a as usize];
|
||||
b -= 1;
|
||||
if !(sa[b as usize] < 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
x ^= 1;
|
||||
}
|
||||
sa[a as usize] = !sa[b as usize];
|
||||
a -= 1;
|
||||
if b <= buf {
|
||||
sa[buf as usize] = t0;
|
||||
break;
|
||||
}
|
||||
sa[b as usize] = sa[a as usize];
|
||||
b -= 1;
|
||||
if x & 2 != 0 {
|
||||
loop {
|
||||
sa[a as usize] = sa[c as usize];
|
||||
a -= 1;
|
||||
sa[c as usize] = sa[a as usize];
|
||||
c -= 1;
|
||||
if !(sa[c as usize] < 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
x ^= 2;
|
||||
}
|
||||
sa[a as usize] = sa[c as usize];
|
||||
a -= 1;
|
||||
sa[c as usize] = sa[a as usize];
|
||||
c -= 1;
|
||||
if c < first {
|
||||
while buf < b {
|
||||
sa[a as usize] = sa[b as usize];
|
||||
a -= 1;
|
||||
sa[b as usize] = sa[a as usize];
|
||||
b -= 1;
|
||||
}
|
||||
sa[a as usize] = sa[b as usize];
|
||||
sa[b as usize] = t0;
|
||||
break;
|
||||
}
|
||||
if sa[b as usize] < 0 {
|
||||
p1 = pa + (!sa[b as usize]) as isize;
|
||||
x |= 1;
|
||||
} else {
|
||||
p1 = pa + sa[b as usize] as isize;
|
||||
}
|
||||
if sa[c as usize] < 0 {
|
||||
p2 = pa + (!sa[c as usize]) as isize;
|
||||
x |= 2;
|
||||
} else {
|
||||
p2 = pa + sa[c as usize] as isize;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `GETIDX` — undoes the "already merged" complement marking.
|
||||
#[inline(always)]
|
||||
fn getidx(a: i32) -> i32 {
|
||||
if 0 <= a {
|
||||
a
|
||||
} else {
|
||||
!a
|
||||
}
|
||||
}
|
||||
|
||||
/// `MERGE_CHECK` — restores or sets the complement marks after a merge.
|
||||
#[inline]
|
||||
fn ss_merge_check(t: &[u8], sa: &mut [i32], pa: isize, a: isize, b: isize, c: i32, depth: i32) {
|
||||
if (c & 1) != 0
|
||||
|| ((c & 2) != 0
|
||||
&& ss_compare_pa(
|
||||
t,
|
||||
sa,
|
||||
pa + getidx(sa[(a - 1) as usize]) as isize,
|
||||
pa + sa[a as usize] as isize,
|
||||
depth,
|
||||
) == 0)
|
||||
{
|
||||
sa[a as usize] = !sa[a as usize];
|
||||
}
|
||||
if (c & 4) != 0
|
||||
&& ss_compare_pa(
|
||||
t,
|
||||
sa,
|
||||
pa + getidx(sa[(b - 1) as usize]) as isize,
|
||||
pa + sa[b as usize] as isize,
|
||||
depth,
|
||||
) == 0
|
||||
{
|
||||
sa[b as usize] = !sa[b as usize];
|
||||
}
|
||||
}
|
||||
|
||||
/* D&C based merge. */
|
||||
fn ss_swapmerge(
|
||||
t: &[u8],
|
||||
sa: &mut [i32],
|
||||
pa: isize,
|
||||
first: isize,
|
||||
middle: isize,
|
||||
last: isize,
|
||||
buf: isize,
|
||||
bufsize: isize,
|
||||
depth: i32,
|
||||
) {
|
||||
let mut stack = [(0isize, 0isize, 0isize, 0i32); SS_SMERGE_STACKSIZE];
|
||||
let mut ssize = 0usize;
|
||||
let mut first = first;
|
||||
let mut middle = middle;
|
||||
let mut last = last;
|
||||
let mut check = 0i32;
|
||||
|
||||
loop {
|
||||
if last - middle <= bufsize {
|
||||
if first < middle && middle < last {
|
||||
ss_mergebackward(t, sa, pa, first, middle, last, buf, depth);
|
||||
}
|
||||
ss_merge_check(t, sa, pa, first, last, check, depth);
|
||||
if ssize == 0 {
|
||||
return;
|
||||
}
|
||||
ssize -= 1;
|
||||
(first, middle, last, check) = stack[ssize];
|
||||
continue;
|
||||
}
|
||||
|
||||
if middle - first <= bufsize {
|
||||
if first < middle {
|
||||
ss_mergeforward(t, sa, pa, first, middle, last, buf, depth);
|
||||
}
|
||||
ss_merge_check(t, sa, pa, first, last, check, depth);
|
||||
if ssize == 0 {
|
||||
return;
|
||||
}
|
||||
ssize -= 1;
|
||||
(first, middle, last, check) = stack[ssize];
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut m: isize = 0;
|
||||
let mut len = std::cmp::min(middle - first, last - middle);
|
||||
let mut half = len >> 1;
|
||||
while 0 < len {
|
||||
if ss_compare_pa(
|
||||
t,
|
||||
sa,
|
||||
pa + getidx(sa[(middle + m + half) as usize]) as isize,
|
||||
pa + getidx(sa[(middle - m - half - 1) as usize]) as isize,
|
||||
depth,
|
||||
) < 0
|
||||
{
|
||||
m += half + 1;
|
||||
half -= (len & 1) ^ 1;
|
||||
}
|
||||
len = half;
|
||||
half >>= 1;
|
||||
}
|
||||
|
||||
if 0 < m {
|
||||
let lm = middle - m;
|
||||
let rm = middle + m;
|
||||
ss_blockswap(sa, lm, middle, m);
|
||||
let mut l = middle;
|
||||
let mut r = middle;
|
||||
let mut next = 0i32;
|
||||
if rm < last {
|
||||
if sa[rm as usize] < 0 {
|
||||
sa[rm as usize] = !sa[rm as usize];
|
||||
if first < lm {
|
||||
loop {
|
||||
l -= 1;
|
||||
if !(sa[l as usize] < 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
next |= 4;
|
||||
}
|
||||
next |= 1;
|
||||
} else if first < lm {
|
||||
while sa[r as usize] < 0 {
|
||||
r += 1;
|
||||
}
|
||||
next |= 2;
|
||||
}
|
||||
}
|
||||
|
||||
if l - first <= last - r {
|
||||
stack[ssize] = (r, rm, last, (next & 3) | (check & 4));
|
||||
ssize += 1;
|
||||
middle = lm;
|
||||
last = l;
|
||||
check = (check & 3) | (next & 4);
|
||||
} else {
|
||||
if (next & 2) != 0 && r == middle {
|
||||
next ^= 6;
|
||||
}
|
||||
stack[ssize] = (first, lm, l, (check & 3) | (next & 4));
|
||||
ssize += 1;
|
||||
first = r;
|
||||
middle = rm;
|
||||
check = (next & 3) | (check & 4);
|
||||
}
|
||||
} else {
|
||||
if ss_compare_pa(
|
||||
t,
|
||||
sa,
|
||||
pa + getidx(sa[(middle - 1) as usize]) as isize,
|
||||
pa + sa[middle as usize] as isize,
|
||||
depth,
|
||||
) == 0
|
||||
{
|
||||
sa[middle as usize] = !sa[middle as usize];
|
||||
}
|
||||
ss_merge_check(t, sa, pa, first, last, check, depth);
|
||||
if ssize == 0 {
|
||||
return;
|
||||
}
|
||||
ssize -= 1;
|
||||
(first, middle, last, check) = stack[ssize];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------------- */
|
||||
|
||||
/* Substring sort */
|
||||
fn sssort(
|
||||
t: &[u8],
|
||||
sa: &mut [i32],
|
||||
pa: isize,
|
||||
first: isize,
|
||||
last: isize,
|
||||
buf: isize,
|
||||
bufsize: isize,
|
||||
depth: i32,
|
||||
n: isize,
|
||||
lastsuffix: bool,
|
||||
) {
|
||||
let mut first = first;
|
||||
let mut buf = buf;
|
||||
let mut bufsize = bufsize;
|
||||
|
||||
if lastsuffix {
|
||||
first += 1;
|
||||
}
|
||||
|
||||
let mut limit: isize = 0;
|
||||
let mut middle = last;
|
||||
if bufsize < SS_BLOCKSIZE && bufsize < last - first {
|
||||
limit = ss_isqrt(last - first);
|
||||
if bufsize < limit {
|
||||
if SS_BLOCKSIZE < limit {
|
||||
limit = SS_BLOCKSIZE;
|
||||
}
|
||||
middle = last - limit;
|
||||
buf = middle;
|
||||
bufsize = limit;
|
||||
} else {
|
||||
limit = 0;
|
||||
}
|
||||
}
|
||||
let mut a = first;
|
||||
let mut i: isize = 0;
|
||||
while SS_BLOCKSIZE < middle - a {
|
||||
ss_mintrosort(t, sa, pa, a, a + SS_BLOCKSIZE, depth);
|
||||
let mut curbufsize = last - (a + SS_BLOCKSIZE);
|
||||
let mut curbuf = a + SS_BLOCKSIZE;
|
||||
if curbufsize <= bufsize {
|
||||
curbufsize = bufsize;
|
||||
curbuf = buf;
|
||||
}
|
||||
let mut b = a;
|
||||
let mut k = SS_BLOCKSIZE;
|
||||
let mut j = i;
|
||||
while j & 1 != 0 {
|
||||
ss_swapmerge(t, sa, pa, b - k, b, b + k, curbuf, curbufsize, depth);
|
||||
b -= k;
|
||||
k <<= 1;
|
||||
j >>= 1;
|
||||
}
|
||||
a += SS_BLOCKSIZE;
|
||||
i += 1;
|
||||
}
|
||||
ss_mintrosort(t, sa, pa, a, middle, depth);
|
||||
let mut k = SS_BLOCKSIZE;
|
||||
while i != 0 {
|
||||
if i & 1 != 0 {
|
||||
ss_swapmerge(t, sa, pa, a - k, a, middle, buf, bufsize, depth);
|
||||
a -= k;
|
||||
}
|
||||
k <<= 1;
|
||||
i >>= 1;
|
||||
}
|
||||
if limit != 0 {
|
||||
ss_mintrosort(t, sa, pa, middle, last, depth);
|
||||
ss_inplacemerge(t, sa, pa, first, middle, last, depth);
|
||||
}
|
||||
|
||||
if lastsuffix {
|
||||
/* Insert last type B* suffix. */
|
||||
let pai0 = sa[(pa + sa[(first - 1) as usize] as isize) as usize];
|
||||
let pai1 = (n - 2) as i32;
|
||||
let i0 = sa[(first - 1) as usize];
|
||||
let mut a = first;
|
||||
while a < last {
|
||||
let av = sa[a as usize];
|
||||
if !(av < 0
|
||||
|| 0 < ss_compare(
|
||||
t,
|
||||
pai0,
|
||||
pai1,
|
||||
sa[(pa + av as isize) as usize],
|
||||
sa[(pa + av as isize + 1) as usize],
|
||||
depth,
|
||||
))
|
||||
{
|
||||
break;
|
||||
}
|
||||
sa[(a - 1) as usize] = av;
|
||||
a += 1;
|
||||
}
|
||||
sa[(a - 1) as usize] = i0;
|
||||
}
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------------- */
|
||||
|
||||
#[inline]
|
||||
fn tr_ilg(n: isize) -> i32 {
|
||||
let n = n as i32;
|
||||
if (n as u32) & 0xffff_0000 != 0 {
|
||||
if (n as u32) & 0xff00_0000 != 0 {
|
||||
24 + LG_TABLE[((n >> 24) & 0xff) as usize]
|
||||
} else {
|
||||
16 + LG_TABLE[((n >> 16) & 0xff) as usize]
|
||||
}
|
||||
} else if n & 0xff00 != 0 {
|
||||
8 + LG_TABLE[((n >> 8) & 0xff) as usize]
|
||||
} else {
|
||||
LG_TABLE[(n & 0xff) as usize]
|
||||
}
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------------- */
|
||||
|
||||
/// `ISAd[SA[p]]` — the depth-offset rank of the suffix stored at `p`.
|
||||
#[inline(always)]
|
||||
fn tr_key(sa: &[i32], isad: isize, p: isize) -> i32 {
|
||||
sa[(isad + sa[p as usize] as isize) as usize]
|
||||
}
|
||||
|
||||
/* Simple insertionsort for small size groups. */
|
||||
fn tr_insertionsort(sa: &mut [i32], isad: isize, first: isize, last: isize) {
|
||||
let mut a = first + 1;
|
||||
while a < last {
|
||||
let t0 = sa[a as usize];
|
||||
let mut b = a - 1;
|
||||
let mut r;
|
||||
loop {
|
||||
r = sa[(isad + t0 as isize) as usize] - tr_key(sa, isad, b);
|
||||
if !(0 > r) {
|
||||
break;
|
||||
}
|
||||
loop {
|
||||
sa[(b + 1) as usize] = sa[b as usize];
|
||||
b -= 1;
|
||||
if !(first <= b && sa[b as usize] < 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if b < first {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if r == 0 {
|
||||
sa[b as usize] = !sa[b as usize];
|
||||
}
|
||||
sa[(b + 1) as usize] = t0;
|
||||
a += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------------- */
|
||||
|
||||
fn tr_fixdown(sa: &mut [i32], isad: isize, base: isize, i: isize, size: isize) {
|
||||
let mut i = i;
|
||||
let v = sa[(base + i) as usize];
|
||||
let c = sa[(isad + v as isize) as usize];
|
||||
loop {
|
||||
let mut j = 2 * i + 1;
|
||||
if j >= size {
|
||||
break;
|
||||
}
|
||||
let mut k = j;
|
||||
j += 1;
|
||||
let mut d = tr_key(sa, isad, base + k);
|
||||
let e = tr_key(sa, isad, base + j);
|
||||
if d < e {
|
||||
k = j;
|
||||
d = e;
|
||||
}
|
||||
if d <= c {
|
||||
break;
|
||||
}
|
||||
sa[(base + i) as usize] = sa[(base + k) as usize];
|
||||
i = k;
|
||||
}
|
||||
sa[(base + i) as usize] = v;
|
||||
}
|
||||
|
||||
/* Simple top-down heapsort. */
|
||||
fn tr_heapsort(sa: &mut [i32], isad: isize, base: isize, size: isize) {
|
||||
let mut m = size;
|
||||
if size % 2 == 0 {
|
||||
m -= 1;
|
||||
if tr_key(sa, isad, base + m / 2) < tr_key(sa, isad, base + m) {
|
||||
sa.swap((base + m) as usize, (base + m / 2) as usize);
|
||||
}
|
||||
}
|
||||
|
||||
let mut i = m / 2 - 1;
|
||||
while 0 <= i {
|
||||
tr_fixdown(sa, isad, base, i, m);
|
||||
i -= 1;
|
||||
}
|
||||
if size % 2 == 0 {
|
||||
sa.swap(base as usize, (base + m) as usize);
|
||||
tr_fixdown(sa, isad, base, 0, m);
|
||||
}
|
||||
let mut i = m - 1;
|
||||
while 0 < i {
|
||||
let t0 = sa[base as usize];
|
||||
sa[base as usize] = sa[(base + i) as usize];
|
||||
tr_fixdown(sa, isad, base, 0, i);
|
||||
sa[(base + i) as usize] = t0;
|
||||
i -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------------- */
|
||||
|
||||
/* Returns the median of three elements. */
|
||||
#[inline]
|
||||
fn tr_median3(sa: &[i32], isad: isize, v1: isize, v2: isize, v3: isize) -> isize {
|
||||
let mut v1 = v1;
|
||||
let mut v2 = v2;
|
||||
if tr_key(sa, isad, v1) > tr_key(sa, isad, v2) {
|
||||
std::mem::swap(&mut v1, &mut v2);
|
||||
}
|
||||
if tr_key(sa, isad, v2) > tr_key(sa, isad, v3) {
|
||||
if tr_key(sa, isad, v1) > tr_key(sa, isad, v3) {
|
||||
return v1;
|
||||
}
|
||||
return v3;
|
||||
}
|
||||
v2
|
||||
}
|
||||
|
||||
/* Returns the median of five elements. */
|
||||
#[inline]
|
||||
fn tr_median5(
|
||||
sa: &[i32],
|
||||
isad: isize,
|
||||
v1: isize,
|
||||
v2: isize,
|
||||
v3: isize,
|
||||
v4: isize,
|
||||
v5: isize,
|
||||
) -> isize {
|
||||
let mut v1 = v1;
|
||||
let mut v2 = v2;
|
||||
let mut v3 = v3;
|
||||
let mut v4 = v4;
|
||||
let mut v5 = v5;
|
||||
if tr_key(sa, isad, v2) > tr_key(sa, isad, v3) {
|
||||
std::mem::swap(&mut v2, &mut v3);
|
||||
}
|
||||
if tr_key(sa, isad, v4) > tr_key(sa, isad, v5) {
|
||||
std::mem::swap(&mut v4, &mut v5);
|
||||
}
|
||||
if tr_key(sa, isad, v2) > tr_key(sa, isad, v4) {
|
||||
std::mem::swap(&mut v2, &mut v4);
|
||||
std::mem::swap(&mut v3, &mut v5);
|
||||
}
|
||||
if tr_key(sa, isad, v1) > tr_key(sa, isad, v3) {
|
||||
std::mem::swap(&mut v1, &mut v3);
|
||||
}
|
||||
if tr_key(sa, isad, v1) > tr_key(sa, isad, v4) {
|
||||
std::mem::swap(&mut v1, &mut v4);
|
||||
std::mem::swap(&mut v3, &mut v5);
|
||||
}
|
||||
if tr_key(sa, isad, v3) > tr_key(sa, isad, v4) {
|
||||
return v4;
|
||||
}
|
||||
v3
|
||||
}
|
||||
|
||||
/* Returns the pivot element. */
|
||||
#[inline]
|
||||
fn tr_pivot(sa: &[i32], isad: isize, first: isize, last: isize) -> isize {
|
||||
let mut t0 = last - first;
|
||||
let middle = first + t0 / 2;
|
||||
|
||||
if t0 <= 512 {
|
||||
if t0 <= 32 {
|
||||
return tr_median3(sa, isad, first, middle, last - 1);
|
||||
}
|
||||
t0 >>= 2;
|
||||
return tr_median5(sa, isad, first, first + t0, middle, last - 1 - t0, last - 1);
|
||||
}
|
||||
t0 >>= 3;
|
||||
let first = tr_median3(sa, isad, first, first + t0, first + (t0 << 1));
|
||||
let middle = tr_median3(sa, isad, middle - t0, middle, middle + t0);
|
||||
let last = tr_median3(sa, isad, last - 1 - (t0 << 1), last - 1 - t0, last - 1);
|
||||
tr_median3(sa, isad, first, middle, last)
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------------- */
|
||||
|
||||
struct TrBudget {
|
||||
chance: i32,
|
||||
remain: i32,
|
||||
incval: i32,
|
||||
count: i32,
|
||||
}
|
||||
|
||||
impl TrBudget {
|
||||
fn new(chance: i32, incval: i32) -> Self {
|
||||
TrBudget {
|
||||
chance,
|
||||
remain: incval,
|
||||
incval,
|
||||
count: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn check(&mut self, size: isize) -> bool {
|
||||
let size = size as i32;
|
||||
if size <= self.remain {
|
||||
self.remain -= size;
|
||||
return true;
|
||||
}
|
||||
if self.chance == 0 {
|
||||
self.count += size;
|
||||
return false;
|
||||
}
|
||||
self.remain += self.incval - size;
|
||||
self.chance -= 1;
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------------- */
|
||||
|
||||
fn tr_partition(
|
||||
sa: &mut [i32],
|
||||
isad: isize,
|
||||
first: isize,
|
||||
middle: isize,
|
||||
last: isize,
|
||||
v: i32,
|
||||
) -> (isize, isize) {
|
||||
let mut first = first;
|
||||
let mut last = last;
|
||||
let mut x: i32 = 0;
|
||||
|
||||
let mut b = middle - 1;
|
||||
loop {
|
||||
b += 1;
|
||||
if !(b < last) {
|
||||
break;
|
||||
}
|
||||
x = tr_key(sa, isad, b);
|
||||
if x != v {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let mut a = b;
|
||||
if a < last && x < v {
|
||||
loop {
|
||||
b += 1;
|
||||
if !(b < last) {
|
||||
break;
|
||||
}
|
||||
x = tr_key(sa, isad, b);
|
||||
if !(x <= v) {
|
||||
break;
|
||||
}
|
||||
if x == v {
|
||||
sa.swap(b as usize, a as usize);
|
||||
a += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut c = last;
|
||||
loop {
|
||||
c -= 1;
|
||||
if !(b < c) {
|
||||
break;
|
||||
}
|
||||
x = tr_key(sa, isad, c);
|
||||
if x != v {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let mut d = c;
|
||||
if b < d && x > v {
|
||||
loop {
|
||||
c -= 1;
|
||||
if !(b < c) {
|
||||
break;
|
||||
}
|
||||
x = tr_key(sa, isad, c);
|
||||
if !(x >= v) {
|
||||
break;
|
||||
}
|
||||
if x == v {
|
||||
sa.swap(c as usize, d as usize);
|
||||
d -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
while b < c {
|
||||
sa.swap(b as usize, c as usize);
|
||||
loop {
|
||||
b += 1;
|
||||
if !(b < c) {
|
||||
break;
|
||||
}
|
||||
x = tr_key(sa, isad, b);
|
||||
if !(x <= v) {
|
||||
break;
|
||||
}
|
||||
if x == v {
|
||||
sa.swap(b as usize, a as usize);
|
||||
a += 1;
|
||||
}
|
||||
}
|
||||
loop {
|
||||
c -= 1;
|
||||
if !(b < c) {
|
||||
break;
|
||||
}
|
||||
x = tr_key(sa, isad, c);
|
||||
if !(x >= v) {
|
||||
break;
|
||||
}
|
||||
if x == v {
|
||||
sa.swap(c as usize, d as usize);
|
||||
d -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if a <= d {
|
||||
c = b - 1;
|
||||
let mut s = a - first;
|
||||
let t0 = b - a;
|
||||
if s > t0 {
|
||||
s = t0;
|
||||
}
|
||||
let mut e = first;
|
||||
let mut f = b - s;
|
||||
while 0 < s {
|
||||
sa.swap(e as usize, f as usize);
|
||||
s -= 1;
|
||||
e += 1;
|
||||
f += 1;
|
||||
}
|
||||
let mut s = d - c;
|
||||
let t0 = last - d - 1;
|
||||
if s > t0 {
|
||||
s = t0;
|
||||
}
|
||||
let mut e = b;
|
||||
let mut f = last - s;
|
||||
while 0 < s {
|
||||
sa.swap(e as usize, f as usize);
|
||||
s -= 1;
|
||||
e += 1;
|
||||
f += 1;
|
||||
}
|
||||
first += b - a;
|
||||
last -= d - c;
|
||||
}
|
||||
(first, last)
|
||||
}
|
||||
|
||||
/* sort suffixes of middle partition by using sorted order of suffixes of
|
||||
* left and right partition. */
|
||||
fn tr_copy(
|
||||
sa: &mut [i32],
|
||||
isa: isize,
|
||||
first: isize,
|
||||
a: isize,
|
||||
b: isize,
|
||||
last: isize,
|
||||
depth: isize,
|
||||
) {
|
||||
/* All cursor arithmetic is relative to the slice start, which is the C
|
||||
* routine's `SA` pointer, so `x - SA` becomes plain `x`. */
|
||||
let v = (b - 1) as i32;
|
||||
|
||||
let mut c = first;
|
||||
let mut d = a - 1;
|
||||
while c <= d {
|
||||
let s = sa[c as usize] - depth as i32;
|
||||
if 0 <= s && sa[(isa + s as isize) as usize] == v {
|
||||
d += 1;
|
||||
sa[d as usize] = s;
|
||||
sa[(isa + s as isize) as usize] = d as i32;
|
||||
}
|
||||
c += 1;
|
||||
}
|
||||
let mut c = last - 1;
|
||||
let e = d + 1;
|
||||
let mut d = b;
|
||||
while e < d {
|
||||
let s = sa[c as usize] - depth as i32;
|
||||
if 0 <= s && sa[(isa + s as isize) as usize] == v {
|
||||
d -= 1;
|
||||
sa[d as usize] = s;
|
||||
sa[(isa + s as isize) as usize] = d as i32;
|
||||
}
|
||||
c -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
fn tr_partialcopy(
|
||||
sa: &mut [i32],
|
||||
isa: isize,
|
||||
first: isize,
|
||||
a: isize,
|
||||
b: isize,
|
||||
last: isize,
|
||||
depth: isize,
|
||||
) {
|
||||
let v = (b - 1) as i32;
|
||||
let mut newrank: i32 = -1;
|
||||
|
||||
let mut lastrank: i32 = -1;
|
||||
let mut c = first;
|
||||
let mut d = a - 1;
|
||||
while c <= d {
|
||||
let s = sa[c as usize] - depth as i32;
|
||||
if 0 <= s && sa[(isa + s as isize) as usize] == v {
|
||||
d += 1;
|
||||
sa[d as usize] = s;
|
||||
let rank = sa[(isa + s as isize + depth) as usize];
|
||||
if lastrank != rank {
|
||||
lastrank = rank;
|
||||
newrank = d as i32;
|
||||
}
|
||||
sa[(isa + s as isize) as usize] = newrank;
|
||||
}
|
||||
c += 1;
|
||||
}
|
||||
|
||||
let mut lastrank: i32 = -1;
|
||||
let mut e = d;
|
||||
while first <= e {
|
||||
let rank = sa[(isa + sa[e as usize] as isize) as usize];
|
||||
if lastrank != rank {
|
||||
lastrank = rank;
|
||||
newrank = e as i32;
|
||||
}
|
||||
if newrank != rank {
|
||||
sa[(isa + sa[e as usize] as isize) as usize] = newrank;
|
||||
}
|
||||
e -= 1;
|
||||
}
|
||||
|
||||
let mut lastrank: i32 = -1;
|
||||
let mut c = last - 1;
|
||||
let e = d + 1;
|
||||
let mut d = b;
|
||||
while e < d {
|
||||
let s = sa[c as usize] - depth as i32;
|
||||
if 0 <= s && sa[(isa + s as isize) as usize] == v {
|
||||
d -= 1;
|
||||
sa[d as usize] = s;
|
||||
let rank = sa[(isa + s as isize + depth) as usize];
|
||||
if lastrank != rank {
|
||||
lastrank = rank;
|
||||
newrank = d as i32;
|
||||
}
|
||||
sa[(isa + s as isize) as usize] = newrank;
|
||||
}
|
||||
c -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
fn tr_introsort(
|
||||
sa: &mut [i32],
|
||||
isa: isize,
|
||||
isad: isize,
|
||||
first: isize,
|
||||
last: isize,
|
||||
budget: &mut TrBudget,
|
||||
) {
|
||||
/* Stack frames are (ISAd, first, last, limit, trlink); the tandem-repeat
|
||||
* copy frame stores its `(a, b)` pair in the pointer fields with a zero
|
||||
* placeholder where C pushes a NULL ISAd. */
|
||||
let mut stack = [(0isize, 0isize, 0isize, 0i32, 0i32); TR_STACKSIZE];
|
||||
let mut ssize = 0usize;
|
||||
let mut trlink: i32 = -1;
|
||||
let mut isad = isad;
|
||||
let mut first = first;
|
||||
let mut last = last;
|
||||
let incr = isad - isa;
|
||||
let mut limit = tr_ilg(last - first);
|
||||
|
||||
loop {
|
||||
if limit < 0 {
|
||||
if limit == -1 {
|
||||
/* tandem repeat partition */
|
||||
let (a, b) = tr_partition(sa, isad - incr, first, first, last, (last - 1) as i32);
|
||||
|
||||
/* update ranks */
|
||||
if a < last {
|
||||
let v = (a - 1) as i32;
|
||||
let mut c = first;
|
||||
while c < a {
|
||||
sa[(isa + sa[c as usize] as isize) as usize] = v;
|
||||
c += 1;
|
||||
}
|
||||
}
|
||||
if b < last {
|
||||
let v = (b - 1) as i32;
|
||||
let mut c = a;
|
||||
while c < b {
|
||||
sa[(isa + sa[c as usize] as isize) as usize] = v;
|
||||
c += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* push */
|
||||
if 1 < b - a {
|
||||
stack[ssize] = (0, a, b, 0, 0);
|
||||
ssize += 1;
|
||||
stack[ssize] = (isad - incr, first, last, -2, trlink);
|
||||
ssize += 1;
|
||||
trlink = ssize as i32 - 2;
|
||||
}
|
||||
if a - first <= last - b {
|
||||
if 1 < a - first {
|
||||
stack[ssize] = (isad, b, last, tr_ilg(last - b), trlink);
|
||||
ssize += 1;
|
||||
last = a;
|
||||
limit = tr_ilg(a - first);
|
||||
} else if 1 < last - b {
|
||||
first = b;
|
||||
limit = tr_ilg(last - b);
|
||||
} else {
|
||||
if ssize == 0 {
|
||||
return;
|
||||
}
|
||||
ssize -= 1;
|
||||
(isad, first, last, limit, trlink) = stack[ssize];
|
||||
}
|
||||
} else if 1 < last - b {
|
||||
stack[ssize] = (isad, first, a, tr_ilg(a - first), trlink);
|
||||
ssize += 1;
|
||||
first = b;
|
||||
limit = tr_ilg(last - b);
|
||||
} else if 1 < a - first {
|
||||
last = a;
|
||||
limit = tr_ilg(a - first);
|
||||
} else {
|
||||
if ssize == 0 {
|
||||
return;
|
||||
}
|
||||
ssize -= 1;
|
||||
(isad, first, last, limit, trlink) = stack[ssize];
|
||||
}
|
||||
} else if limit == -2 {
|
||||
/* tandem repeat copy */
|
||||
ssize -= 1;
|
||||
let a = stack[ssize].1;
|
||||
let b = stack[ssize].2;
|
||||
if stack[ssize].3 == 0 {
|
||||
tr_copy(sa, isa, first, a, b, last, isad - isa);
|
||||
} else {
|
||||
if 0 <= trlink {
|
||||
stack[trlink as usize].3 = -1;
|
||||
}
|
||||
tr_partialcopy(sa, isa, first, a, b, last, isad - isa);
|
||||
}
|
||||
if ssize == 0 {
|
||||
return;
|
||||
}
|
||||
ssize -= 1;
|
||||
(isad, first, last, limit, trlink) = stack[ssize];
|
||||
} else {
|
||||
/* sorted partition */
|
||||
if 0 <= sa[first as usize] {
|
||||
let mut a = first;
|
||||
loop {
|
||||
sa[(isa + sa[a as usize] as isize) as usize] = a as i32;
|
||||
a += 1;
|
||||
if !(a < last && 0 <= sa[a as usize]) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
first = a;
|
||||
}
|
||||
if first < last {
|
||||
let mut a = first;
|
||||
loop {
|
||||
sa[a as usize] = !sa[a as usize];
|
||||
a += 1;
|
||||
if !(sa[a as usize] < 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let next =
|
||||
if sa[(isa + sa[a as usize] as isize) as usize] != tr_key(sa, isad, a) {
|
||||
tr_ilg(a - first + 1)
|
||||
} else {
|
||||
-1
|
||||
};
|
||||
a += 1;
|
||||
if a < last {
|
||||
let v = (a - 1) as i32;
|
||||
let mut b = first;
|
||||
while b < a {
|
||||
sa[(isa + sa[b as usize] as isize) as usize] = v;
|
||||
b += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* push */
|
||||
if budget.check(a - first) {
|
||||
if a - first <= last - a {
|
||||
stack[ssize] = (isad, a, last, -3, trlink);
|
||||
ssize += 1;
|
||||
isad += incr;
|
||||
last = a;
|
||||
limit = next;
|
||||
} else if 1 < last - a {
|
||||
stack[ssize] = (isad + incr, first, a, next, trlink);
|
||||
ssize += 1;
|
||||
first = a;
|
||||
limit = -3;
|
||||
} else {
|
||||
isad += incr;
|
||||
last = a;
|
||||
limit = next;
|
||||
}
|
||||
} else {
|
||||
if 0 <= trlink {
|
||||
stack[trlink as usize].3 = -1;
|
||||
}
|
||||
if 1 < last - a {
|
||||
first = a;
|
||||
limit = -3;
|
||||
} else {
|
||||
if ssize == 0 {
|
||||
return;
|
||||
}
|
||||
ssize -= 1;
|
||||
(isad, first, last, limit, trlink) = stack[ssize];
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if ssize == 0 {
|
||||
return;
|
||||
}
|
||||
ssize -= 1;
|
||||
(isad, first, last, limit, trlink) = stack[ssize];
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if last - first <= TR_INSERTIONSORT_THRESHOLD {
|
||||
tr_insertionsort(sa, isad, first, last);
|
||||
limit = -3;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* C decrements `limit` here (`limit-- == 0`); the decrement is
|
||||
* observable only on the not-taken path because the taken path
|
||||
* overwrites `limit` with -3. */
|
||||
if limit == 0 {
|
||||
tr_heapsort(sa, isad, first, last - first);
|
||||
let mut a = last - 1;
|
||||
while first < a {
|
||||
let x = tr_key(sa, isad, a);
|
||||
let mut b = a - 1;
|
||||
while first <= b && tr_key(sa, isad, b) == x {
|
||||
sa[b as usize] = !sa[b as usize];
|
||||
b -= 1;
|
||||
}
|
||||
a = b;
|
||||
}
|
||||
limit = -3;
|
||||
continue;
|
||||
}
|
||||
limit -= 1;
|
||||
|
||||
/* choose pivot */
|
||||
let a = tr_pivot(sa, isad, first, last);
|
||||
sa.swap(first as usize, a as usize);
|
||||
let v = tr_key(sa, isad, first);
|
||||
|
||||
/* partition */
|
||||
let (a, b) = tr_partition(sa, isad, first, first + 1, last, v);
|
||||
if last - first != b - a {
|
||||
let next = if sa[(isa + sa[a as usize] as isize) as usize] != v {
|
||||
tr_ilg(b - a)
|
||||
} else {
|
||||
-1
|
||||
};
|
||||
|
||||
/* update ranks */
|
||||
{
|
||||
let vv = (a - 1) as i32;
|
||||
let mut c = first;
|
||||
while c < a {
|
||||
sa[(isa + sa[c as usize] as isize) as usize] = vv;
|
||||
c += 1;
|
||||
}
|
||||
}
|
||||
if b < last {
|
||||
let vv = (b - 1) as i32;
|
||||
let mut c = a;
|
||||
while c < b {
|
||||
sa[(isa + sa[c as usize] as isize) as usize] = vv;
|
||||
c += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* push */
|
||||
if 1 < b - a && budget.check(b - a) {
|
||||
if a - first <= last - b {
|
||||
if last - b <= b - a {
|
||||
if 1 < a - first {
|
||||
stack[ssize] = (isad + incr, a, b, next, trlink);
|
||||
ssize += 1;
|
||||
stack[ssize] = (isad, b, last, limit, trlink);
|
||||
ssize += 1;
|
||||
last = a;
|
||||
} else if 1 < last - b {
|
||||
stack[ssize] = (isad + incr, a, b, next, trlink);
|
||||
ssize += 1;
|
||||
first = b;
|
||||
} else {
|
||||
isad += incr;
|
||||
first = a;
|
||||
last = b;
|
||||
limit = next;
|
||||
}
|
||||
} else if a - first <= b - a {
|
||||
if 1 < a - first {
|
||||
stack[ssize] = (isad, b, last, limit, trlink);
|
||||
ssize += 1;
|
||||
stack[ssize] = (isad + incr, a, b, next, trlink);
|
||||
ssize += 1;
|
||||
last = a;
|
||||
} else {
|
||||
stack[ssize] = (isad, b, last, limit, trlink);
|
||||
ssize += 1;
|
||||
isad += incr;
|
||||
first = a;
|
||||
last = b;
|
||||
limit = next;
|
||||
}
|
||||
} else {
|
||||
stack[ssize] = (isad, b, last, limit, trlink);
|
||||
ssize += 1;
|
||||
stack[ssize] = (isad, first, a, limit, trlink);
|
||||
ssize += 1;
|
||||
isad += incr;
|
||||
first = a;
|
||||
last = b;
|
||||
limit = next;
|
||||
}
|
||||
} else if a - first <= b - a {
|
||||
if 1 < last - b {
|
||||
stack[ssize] = (isad + incr, a, b, next, trlink);
|
||||
ssize += 1;
|
||||
stack[ssize] = (isad, first, a, limit, trlink);
|
||||
ssize += 1;
|
||||
first = b;
|
||||
} else if 1 < a - first {
|
||||
stack[ssize] = (isad + incr, a, b, next, trlink);
|
||||
ssize += 1;
|
||||
last = a;
|
||||
} else {
|
||||
isad += incr;
|
||||
first = a;
|
||||
last = b;
|
||||
limit = next;
|
||||
}
|
||||
} else if last - b <= b - a {
|
||||
if 1 < last - b {
|
||||
stack[ssize] = (isad, first, a, limit, trlink);
|
||||
ssize += 1;
|
||||
stack[ssize] = (isad + incr, a, b, next, trlink);
|
||||
ssize += 1;
|
||||
first = b;
|
||||
} else {
|
||||
stack[ssize] = (isad, first, a, limit, trlink);
|
||||
ssize += 1;
|
||||
isad += incr;
|
||||
first = a;
|
||||
last = b;
|
||||
limit = next;
|
||||
}
|
||||
} else {
|
||||
stack[ssize] = (isad, first, a, limit, trlink);
|
||||
ssize += 1;
|
||||
stack[ssize] = (isad, b, last, limit, trlink);
|
||||
ssize += 1;
|
||||
isad += incr;
|
||||
first = a;
|
||||
last = b;
|
||||
limit = next;
|
||||
}
|
||||
} else {
|
||||
if 1 < b - a && 0 <= trlink {
|
||||
stack[trlink as usize].3 = -1;
|
||||
}
|
||||
if a - first <= last - b {
|
||||
if 1 < a - first {
|
||||
stack[ssize] = (isad, b, last, limit, trlink);
|
||||
ssize += 1;
|
||||
last = a;
|
||||
} else if 1 < last - b {
|
||||
first = b;
|
||||
} else {
|
||||
if ssize == 0 {
|
||||
return;
|
||||
}
|
||||
ssize -= 1;
|
||||
(isad, first, last, limit, trlink) = stack[ssize];
|
||||
}
|
||||
} else if 1 < last - b {
|
||||
stack[ssize] = (isad, first, a, limit, trlink);
|
||||
ssize += 1;
|
||||
first = b;
|
||||
} else if 1 < a - first {
|
||||
last = a;
|
||||
} else {
|
||||
if ssize == 0 {
|
||||
return;
|
||||
}
|
||||
ssize -= 1;
|
||||
(isad, first, last, limit, trlink) = stack[ssize];
|
||||
}
|
||||
}
|
||||
} else if budget.check(last - first) {
|
||||
limit = tr_ilg(last - first);
|
||||
isad += incr;
|
||||
} else {
|
||||
if 0 <= trlink {
|
||||
stack[trlink as usize].3 = -1;
|
||||
}
|
||||
if ssize == 0 {
|
||||
return;
|
||||
}
|
||||
ssize -= 1;
|
||||
(isad, first, last, limit, trlink) = stack[ssize];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------------- */
|
||||
|
||||
/* Tandem repeat sort */
|
||||
fn trsort(sa: &mut [i32], isa: isize, n: isize, depth: isize) {
|
||||
let mut budget = TrBudget::new(tr_ilg(n) * 2 / 3, n as i32);
|
||||
/* trbudget_init(&budget, tr_ilg(n) * 3 / 4, n); */
|
||||
let mut isad = isa + depth;
|
||||
while -(n as i32) < sa[0] {
|
||||
let mut first: isize = 0;
|
||||
let mut skip: isize = 0;
|
||||
let mut unsorted: i32 = 0;
|
||||
loop {
|
||||
let t0 = sa[first as usize];
|
||||
if t0 < 0 {
|
||||
first -= t0 as isize;
|
||||
skip += t0 as isize;
|
||||
} else {
|
||||
if skip != 0 {
|
||||
sa[(first + skip) as usize] = skip as i32;
|
||||
skip = 0;
|
||||
}
|
||||
let last = sa[(isa + t0 as isize) as usize] as isize + 1;
|
||||
if 1 < last - first {
|
||||
budget.count = 0;
|
||||
tr_introsort(sa, isa, isad, first, last, &mut budget);
|
||||
if budget.count != 0 {
|
||||
unsorted += budget.count;
|
||||
} else {
|
||||
skip = first - last;
|
||||
}
|
||||
} else if last - first == 1 {
|
||||
skip = -1;
|
||||
}
|
||||
first = last;
|
||||
}
|
||||
if !(first < n) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if skip != 0 {
|
||||
sa[(first + skip) as usize] = skip as i32;
|
||||
}
|
||||
if unsorted == 0 {
|
||||
break;
|
||||
}
|
||||
isad += isad - isa;
|
||||
}
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------------- */
|
||||
|
||||
/// `BUCKET_B(c0, c1)` for the 256-symbol alphabet.
|
||||
#[inline(always)]
|
||||
fn bb(c0: i32, c1: i32) -> usize {
|
||||
(((c1 as u32) << 8) | c0 as u32) as usize
|
||||
}
|
||||
|
||||
/// `BUCKET_BSTAR(c0, c1)` for the 256-symbol alphabet.
|
||||
#[inline(always)]
|
||||
fn bstar(c0: i32, c1: i32) -> usize {
|
||||
(((c0 as u32) << 8) | c1 as u32) as usize
|
||||
}
|
||||
|
||||
/* Sorts suffixes of type B*. */
|
||||
fn sort_type_bstar(
|
||||
t: &[u8],
|
||||
sa: &mut [i32],
|
||||
bucket_a: &mut [i32],
|
||||
bucket_b: &mut [i32],
|
||||
n: isize,
|
||||
) -> isize {
|
||||
/* Initialize bucket arrays. */
|
||||
for slot in bucket_a.iter_mut() {
|
||||
*slot = 0;
|
||||
}
|
||||
for slot in bucket_b.iter_mut() {
|
||||
*slot = 0;
|
||||
}
|
||||
|
||||
/* Count the number of occurrences of the first one or two characters of
|
||||
each type A, B and B* suffix. Moreover, store the beginning position of
|
||||
all type B* suffixes into the array SA. */
|
||||
let mut i = n - 1;
|
||||
let mut m = n;
|
||||
let mut c0 = t[(n - 1) as usize] as i32;
|
||||
let mut c1;
|
||||
while 0 <= i {
|
||||
/* type A suffix. */
|
||||
loop {
|
||||
c1 = c0;
|
||||
bucket_a[c1 as usize] += 1;
|
||||
i -= 1;
|
||||
if 0 <= i {
|
||||
c0 = t[i as usize] as i32;
|
||||
if c0 >= c1 {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
if 0 <= i {
|
||||
/* type B* suffix. */
|
||||
bucket_b[bstar(c0, c1)] += 1;
|
||||
m -= 1;
|
||||
sa[m as usize] = i as i32;
|
||||
/* type B suffix. */
|
||||
i -= 1;
|
||||
c1 = c0;
|
||||
while 0 <= i {
|
||||
c0 = t[i as usize] as i32;
|
||||
if !(c0 <= c1) {
|
||||
break;
|
||||
}
|
||||
bucket_b[bb(c0, c1)] += 1;
|
||||
i -= 1;
|
||||
c1 = c0;
|
||||
}
|
||||
}
|
||||
}
|
||||
let m = n - m;
|
||||
/*
|
||||
note:
|
||||
A type B* suffix is lexicographically smaller than a type B suffix that
|
||||
begins with the same first two characters.
|
||||
*/
|
||||
|
||||
/* Calculate the index of start/end point of each bucket. */
|
||||
{
|
||||
let mut i: i32 = 0;
|
||||
let mut j: i32 = 0;
|
||||
for c0 in 0..ALPHABET_SIZE {
|
||||
let t0 = i + bucket_a[c0 as usize];
|
||||
bucket_a[c0 as usize] = i + j; /* start point */
|
||||
i = t0 + bucket_b[bb(c0, c0)];
|
||||
for c1 in (c0 + 1)..ALPHABET_SIZE {
|
||||
j += bucket_b[bstar(c0, c1)];
|
||||
bucket_b[bstar(c0, c1)] = j; /* end point */
|
||||
i += bucket_b[bb(c0, c1)];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if 0 < m {
|
||||
/* Sort the type B* suffixes by their first two characters. */
|
||||
let pab = n - m;
|
||||
let isab = m;
|
||||
let mut i = m - 2;
|
||||
while 0 <= i {
|
||||
let t0 = sa[(pab + i) as usize];
|
||||
let c0 = t[t0 as usize] as i32;
|
||||
let c1 = t[(t0 + 1) as usize] as i32;
|
||||
bucket_b[bstar(c0, c1)] -= 1;
|
||||
sa[bucket_b[bstar(c0, c1)] as usize] = i as i32;
|
||||
i -= 1;
|
||||
}
|
||||
{
|
||||
let t0 = sa[(pab + m - 1) as usize];
|
||||
let c0 = t[t0 as usize] as i32;
|
||||
let c1 = t[(t0 + 1) as usize] as i32;
|
||||
bucket_b[bstar(c0, c1)] -= 1;
|
||||
sa[bucket_b[bstar(c0, c1)] as usize] = (m - 1) as i32;
|
||||
}
|
||||
|
||||
/* Sort the type B* substrings using sssort. */
|
||||
let buf = m;
|
||||
let bufsize = n - 2 * m;
|
||||
let mut c0 = ALPHABET_SIZE - 2;
|
||||
let mut j = m;
|
||||
while 0 < j {
|
||||
let mut c1 = ALPHABET_SIZE - 1;
|
||||
while c0 < c1 {
|
||||
let i = bucket_b[bstar(c0, c1)] as isize;
|
||||
if 1 < j - i {
|
||||
sssort(
|
||||
t,
|
||||
sa,
|
||||
pab,
|
||||
i,
|
||||
j,
|
||||
buf,
|
||||
bufsize,
|
||||
2,
|
||||
n,
|
||||
sa[i as usize] == (m - 1) as i32,
|
||||
);
|
||||
}
|
||||
j = i;
|
||||
c1 -= 1;
|
||||
}
|
||||
c0 -= 1;
|
||||
}
|
||||
|
||||
/* Compute ranks of type B* substrings. */
|
||||
let mut i = m - 1;
|
||||
while 0 <= i {
|
||||
if 0 <= sa[i as usize] {
|
||||
let j = i;
|
||||
loop {
|
||||
sa[(isab + sa[i as usize] as isize) as usize] = i as i32;
|
||||
i -= 1;
|
||||
if !(0 <= i && 0 <= sa[i as usize]) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
sa[(i + 1) as usize] = (i - j) as i32;
|
||||
if i <= 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let j = i;
|
||||
loop {
|
||||
sa[i as usize] = !sa[i as usize];
|
||||
sa[(isab + sa[i as usize] as isize) as usize] = j as i32;
|
||||
i -= 1;
|
||||
if !(sa[i as usize] < 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
sa[(isab + sa[i as usize] as isize) as usize] = j as i32;
|
||||
i -= 1;
|
||||
}
|
||||
|
||||
/* Construct the inverse suffix array of type B* suffixes using
|
||||
trsort. */
|
||||
trsort(sa, isab, m, 1);
|
||||
|
||||
/* Set the sorted order of type B* suffixes. */
|
||||
let mut i = n - 1;
|
||||
let mut j = m;
|
||||
let mut c0 = t[(n - 1) as usize] as i32;
|
||||
while 0 <= i {
|
||||
i -= 1;
|
||||
let mut c1 = c0;
|
||||
while 0 <= i {
|
||||
c0 = t[i as usize] as i32;
|
||||
if !(c0 >= c1) {
|
||||
break;
|
||||
}
|
||||
i -= 1;
|
||||
c1 = c0;
|
||||
}
|
||||
if 0 <= i {
|
||||
let t0 = i;
|
||||
i -= 1;
|
||||
c1 = c0;
|
||||
while 0 <= i {
|
||||
c0 = t[i as usize] as i32;
|
||||
if !(c0 <= c1) {
|
||||
break;
|
||||
}
|
||||
i -= 1;
|
||||
c1 = c0;
|
||||
}
|
||||
j -= 1;
|
||||
sa[sa[(isab + j) as usize] as usize] = if t0 == 0 || 1 < t0 - i {
|
||||
t0 as i32
|
||||
} else {
|
||||
!(t0 as i32)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/* Calculate the index of start/end point of each bucket. */
|
||||
bucket_b[bb(ALPHABET_SIZE - 1, ALPHABET_SIZE - 1)] = n as i32; /* end point */
|
||||
let mut k = m - 1;
|
||||
let mut c0 = ALPHABET_SIZE - 2;
|
||||
while 0 <= c0 {
|
||||
let mut i = bucket_a[(c0 + 1) as usize] as isize - 1;
|
||||
let mut c1 = ALPHABET_SIZE - 1;
|
||||
while c0 < c1 {
|
||||
let t0 = i - bucket_b[bb(c0, c1)] as isize;
|
||||
bucket_b[bb(c0, c1)] = i as i32; /* end point */
|
||||
|
||||
/* Move all type B* suffixes to the correct position. */
|
||||
i = t0;
|
||||
let j = bucket_b[bstar(c0, c1)] as isize;
|
||||
while j <= k {
|
||||
sa[i as usize] = sa[k as usize];
|
||||
i -= 1;
|
||||
k -= 1;
|
||||
}
|
||||
c1 -= 1;
|
||||
}
|
||||
bucket_b[bstar(c0, c0 + 1)] = (i - bucket_b[bb(c0, c0)] as isize + 1) as i32; /* start point */
|
||||
bucket_b[bb(c0, c0)] = i as i32; /* end point */
|
||||
c0 -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
m
|
||||
}
|
||||
|
||||
/* Constructs the suffix array by using the sorted order of type B*
|
||||
* suffixes. */
|
||||
fn construct_sa(
|
||||
t: &[u8],
|
||||
sa: &mut [i32],
|
||||
bucket_a: &mut [i32],
|
||||
bucket_b: &mut [i32],
|
||||
n: isize,
|
||||
m: isize,
|
||||
) {
|
||||
if 0 < m {
|
||||
/* Construct the sorted order of type B suffixes by using
|
||||
the sorted order of type B* suffixes. */
|
||||
let mut c1 = ALPHABET_SIZE - 2;
|
||||
while 0 <= c1 {
|
||||
/* Scan the suffix array from right to left. */
|
||||
let i = bucket_b[bstar(c1, c1 + 1)] as isize;
|
||||
let mut j = bucket_a[(c1 + 1) as usize] as isize - 1;
|
||||
let mut k: isize = 0;
|
||||
let mut c2: i32 = -1;
|
||||
while i <= j {
|
||||
let mut s = sa[j as usize];
|
||||
if 0 < s {
|
||||
debug_assert_eq!(t[s as usize] as i32, c1);
|
||||
debug_assert!((s as isize + 1) < n && t[s as usize] <= t[(s + 1) as usize]);
|
||||
debug_assert!(t[(s - 1) as usize] <= t[s as usize]);
|
||||
sa[j as usize] = !s;
|
||||
s -= 1;
|
||||
let c0 = t[s as usize] as i32;
|
||||
if 0 < s && (t[(s - 1) as usize] as i32) > c0 {
|
||||
s = !s;
|
||||
}
|
||||
if c0 != c2 {
|
||||
if 0 <= c2 {
|
||||
bucket_b[bb(c2, c1)] = k as i32;
|
||||
}
|
||||
c2 = c0;
|
||||
k = bucket_b[bb(c2, c1)] as isize;
|
||||
}
|
||||
debug_assert!(k < j);
|
||||
sa[k as usize] = s;
|
||||
k -= 1;
|
||||
} else {
|
||||
debug_assert!((s == 0 && t[s as usize] as i32 == c1) || s < 0);
|
||||
sa[j as usize] = !s;
|
||||
}
|
||||
j -= 1;
|
||||
}
|
||||
c1 -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* Construct the suffix array by using the sorted order of type B
|
||||
suffixes. */
|
||||
let mut c2 = t[(n - 1) as usize] as i32;
|
||||
let mut k = bucket_a[c2 as usize] as isize;
|
||||
sa[k as usize] = if (t[(n - 2) as usize] as i32) < c2 {
|
||||
!((n - 1) as i32)
|
||||
} else {
|
||||
(n - 1) as i32
|
||||
};
|
||||
k += 1;
|
||||
/* Scan the suffix array from left to right. */
|
||||
let mut i: isize = 0;
|
||||
let j = n;
|
||||
while i < j {
|
||||
let mut s = sa[i as usize];
|
||||
if 0 < s {
|
||||
debug_assert!(t[(s - 1) as usize] >= t[s as usize]);
|
||||
s -= 1;
|
||||
let c0 = t[s as usize] as i32;
|
||||
if s == 0 || (t[(s - 1) as usize] as i32) < c0 {
|
||||
s = !s;
|
||||
}
|
||||
if c0 != c2 {
|
||||
bucket_a[c2 as usize] = k as i32;
|
||||
c2 = c0;
|
||||
k = bucket_a[c2 as usize] as isize;
|
||||
}
|
||||
debug_assert!(i < k);
|
||||
sa[k as usize] = s;
|
||||
k += 1;
|
||||
} else {
|
||||
debug_assert!(s < 0);
|
||||
sa[i as usize] = !s;
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------------- */
|
||||
|
||||
/// Rust implementation of the `divsufsort()` entry point used by
|
||||
/// `ZDICT_trainFromBuffer_legacy()`.
|
||||
///
|
||||
/// Integration removes the C function body, so this direct export provides
|
||||
/// the existing library symbol without a wrapper. The `open_mp` parameter is
|
||||
/// accepted for signature compatibility only: zstd never defines
|
||||
/// `LIBBSC_OPENMP`, so the C implementation ignored it as well.
|
||||
///
|
||||
/// Returns 0 on success, -1 for invalid arguments, and -2 when the bucket
|
||||
/// work arrays cannot be allocated, exactly like the C routine.
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn divsufsort(
|
||||
t: *const u8,
|
||||
sa: *mut c_int,
|
||||
n: c_int,
|
||||
open_mp: c_int,
|
||||
) -> c_int {
|
||||
let _ = open_mp;
|
||||
|
||||
/* Check arguments. */
|
||||
if t.is_null() || sa.is_null() || n < 0 {
|
||||
return -1;
|
||||
}
|
||||
if n == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let text = unsafe { slice::from_raw_parts(t, n as usize) };
|
||||
let suffix = unsafe { slice::from_raw_parts_mut(sa, n as usize) };
|
||||
if n == 1 {
|
||||
suffix[0] = 0;
|
||||
return 0;
|
||||
}
|
||||
if n == 2 {
|
||||
let m = usize::from(text[0] < text[1]);
|
||||
suffix[m ^ 1] = 0;
|
||||
suffix[m] = 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
let mut bucket_a: Vec<i32> = Vec::new();
|
||||
let mut bucket_b: Vec<i32> = Vec::new();
|
||||
if bucket_a.try_reserve_exact(BUCKET_A_SIZE).is_err()
|
||||
|| bucket_b.try_reserve_exact(BUCKET_B_SIZE).is_err()
|
||||
{
|
||||
/* Match the C implementation's -2 result when malloc fails. */
|
||||
return -2;
|
||||
}
|
||||
bucket_a.resize(BUCKET_A_SIZE, 0);
|
||||
bucket_b.resize(BUCKET_B_SIZE, 0);
|
||||
|
||||
/* Suffixsort. */
|
||||
let m = sort_type_bstar(text, suffix, &mut bucket_a, &mut bucket_b, n as isize);
|
||||
construct_sa(text, suffix, &mut bucket_a, &mut bucket_b, n as isize, m);
|
||||
0
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::ptr;
|
||||
|
||||
fn build_sa(text: &[u8]) -> Vec<i32> {
|
||||
let mut sa = vec![0i32; text.len()];
|
||||
let result = unsafe { divsufsort(text.as_ptr(), sa.as_mut_ptr(), text.len() as c_int, 0) };
|
||||
assert_eq!(result, 0);
|
||||
sa
|
||||
}
|
||||
|
||||
/// Trivial O(n^2 log n) reference: sort the suffix start positions by the
|
||||
/// suffixes themselves.
|
||||
fn reference_sa(text: &[u8]) -> Vec<i32> {
|
||||
let mut sa: Vec<i32> = (0..text.len() as i32).collect();
|
||||
sa.sort_by(|&a, &b| text[a as usize..].cmp(&text[b as usize..]));
|
||||
sa
|
||||
}
|
||||
|
||||
/// Suffix-array invariants: a permutation of `0..n` whose suffixes are in
|
||||
/// strictly increasing lexicographic order.
|
||||
fn assert_valid_sa(text: &[u8], sa: &[i32]) {
|
||||
assert_eq!(sa.len(), text.len());
|
||||
let mut seen = vec![false; text.len()];
|
||||
for &p in sa {
|
||||
let p = usize::try_from(p).expect("suffix index must be non-negative");
|
||||
assert!(p < text.len(), "suffix index {p} out of range");
|
||||
assert!(!seen[p], "duplicate suffix index {p}");
|
||||
seen[p] = true;
|
||||
}
|
||||
for pair in sa.windows(2) {
|
||||
assert!(
|
||||
text[pair[0] as usize..] < text[pair[1] as usize..],
|
||||
"suffixes {} and {} are not in sorted order",
|
||||
pair[0],
|
||||
pair[1]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Fixed-seed numerical-recipes LCG, used to generate reproducible
|
||||
/// pseudo-random sample buffers.
|
||||
fn lcg_bytes(len: usize, seed: u32, alphabet: u32) -> Vec<u8> {
|
||||
let mut state = seed;
|
||||
(0..len)
|
||||
.map(|_| {
|
||||
state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
|
||||
((state >> 24) % alphabet) as u8
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_invalid_arguments() {
|
||||
let text = [0u8; 1];
|
||||
let mut sa = [0i32; 1];
|
||||
assert_eq!(
|
||||
unsafe { divsufsort(ptr::null(), sa.as_mut_ptr(), 1, 0) },
|
||||
-1
|
||||
);
|
||||
assert_eq!(
|
||||
unsafe { divsufsort(text.as_ptr(), ptr::null_mut(), 1, 0) },
|
||||
-1
|
||||
);
|
||||
assert_eq!(
|
||||
unsafe { divsufsort(text.as_ptr(), sa.as_mut_ptr(), -1, 0) },
|
||||
-1
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sorts_trivial_inputs() {
|
||||
/* empty */
|
||||
let text = [0u8; 1];
|
||||
let mut sa = [i32::MIN; 1];
|
||||
assert_eq!(
|
||||
unsafe { divsufsort(text.as_ptr(), sa.as_mut_ptr(), 0, 0) },
|
||||
0
|
||||
);
|
||||
assert_eq!(sa[0], i32::MIN, "n == 0 must not touch the output");
|
||||
|
||||
/* single byte */
|
||||
assert_eq!(build_sa(b"z"), [0]);
|
||||
|
||||
/* two bytes: ascending, descending, and equal */
|
||||
assert_eq!(build_sa(b"ab"), [0, 1]);
|
||||
assert_eq!(build_sa(b"ba"), [1, 0]);
|
||||
assert_eq!(build_sa(b"aa"), [1, 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sorts_all_equal_bytes() {
|
||||
let text = vec![b'q'; 10_000];
|
||||
let sa = build_sa(&text);
|
||||
/* For a constant text the shortest suffix sorts first. */
|
||||
let expected: Vec<i32> = (0..text.len() as i32).rev().collect();
|
||||
assert_eq!(sa, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sorts_abracadabra_exactly() {
|
||||
/* Hand-computed: a(10) abra(7) abracadabra(0) acadabra(3) adabra(5)
|
||||
* bra(8) bracadabra(1) cadabra(4) dabra(6) ra(9) racadabra(2). */
|
||||
assert_eq!(build_sa(b"abracadabra"), [10, 7, 0, 3, 5, 8, 1, 4, 6, 9, 2]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_reference_on_periodic_text() {
|
||||
/* Tandem repeats exercise trsort's repeat partitioning. */
|
||||
let text: Vec<u8> = b"ab".iter().copied().cycle().take(4096).collect();
|
||||
let sa = build_sa(&text);
|
||||
assert_valid_sa(&text, &sa);
|
||||
assert_eq!(sa, reference_sa(&text));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_reference_on_random_bytes() {
|
||||
let text = lcg_bytes(8192, 0x0BAD_5EED, 256);
|
||||
let sa = build_sa(&text);
|
||||
assert_valid_sa(&text, &sa);
|
||||
assert_eq!(sa, reference_sa(&text));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_reference_on_low_alphabet_text() {
|
||||
/* A four-symbol alphabet produces the large first-two-character
|
||||
* buckets that reach sssort's block merging and the deeper trsort
|
||||
* paths. */
|
||||
let text = lcg_bytes(16_384, 0xDEAD_BEEF, 4);
|
||||
let sa = build_sa(&text);
|
||||
assert_valid_sa(&text, &sa);
|
||||
assert_eq!(sa, reference_sa(&text));
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -5,8 +5,6 @@ pub mod bitstream;
|
||||
pub mod common;
|
||||
pub mod cpu;
|
||||
pub mod debug;
|
||||
#[cfg(feature = "dict-builder")]
|
||||
pub mod divsufsort;
|
||||
pub mod entropy_common;
|
||||
pub mod errors;
|
||||
#[cfg(feature = "compression")]
|
||||
@@ -34,6 +32,8 @@ pub mod zstd_compress_params;
|
||||
#[cfg(feature = "compression")]
|
||||
pub mod zstd_compress_sequences;
|
||||
#[cfg(feature = "compression")]
|
||||
pub mod zstd_compress_stats;
|
||||
#[cfg(feature = "compression")]
|
||||
pub mod zstd_compress_superblock;
|
||||
#[cfg(feature = "decompression")]
|
||||
pub mod zstd_ddict;
|
||||
|
||||
+4
-47
@@ -210,7 +210,6 @@ struct Cli {
|
||||
mmap_dict: i32,
|
||||
progress: i32,
|
||||
workers: Option<i32>,
|
||||
single_thread: bool,
|
||||
block_size: Option<usize>,
|
||||
mem_limit: Option<u32>,
|
||||
ldm: bool,
|
||||
@@ -255,7 +254,6 @@ impl Cli {
|
||||
mmap_dict: ZSTD_PS_AUTO,
|
||||
progress: FIO_PS_AUTO,
|
||||
workers: None,
|
||||
single_thread: false,
|
||||
block_size: None,
|
||||
mem_limit: None,
|
||||
ldm: false,
|
||||
@@ -334,11 +332,8 @@ unsafe fn default_worker_count() -> i32 {
|
||||
}
|
||||
|
||||
#[cfg(feature = "compression")]
|
||||
unsafe fn resolved_worker_count(workers: Option<i32>, single_thread: bool) -> i32 {
|
||||
unsafe fn resolved_worker_count(workers: Option<i32>) -> i32 {
|
||||
match workers {
|
||||
/* --single-thread pins zero workers; a bare zero (-T0 or the zstdmt
|
||||
* program name) auto-detects the core count as in the C CLI. */
|
||||
Some(0) if single_thread => 0,
|
||||
Some(0) => unsafe { UTIL_countPhysicalCores() }.max(1),
|
||||
Some(workers) => workers,
|
||||
None => unsafe { default_worker_count() },
|
||||
@@ -390,7 +385,7 @@ fn usage(advanced: bool) {
|
||||
let _ = writeln!(out, "\nImplemented advanced compression controls:");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
" --fast[=#], --ultra, --long[=#], --threads=#, --single-thread, --block-size=#"
|
||||
" --fast[=#], --ultra, --long[=#], --threads=#, --block-size=#"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
@@ -643,7 +638,6 @@ fn parse_long_option(
|
||||
| "--no-row-match-finder"
|
||||
| "--row-match-finder"
|
||||
| "--rsyncable"
|
||||
| "--single-thread"
|
||||
| "--compress-literals"
|
||||
| "--no-compress-literals"
|
||||
| "--exclude-compressed"
|
||||
@@ -815,14 +809,6 @@ fn parse_long_option(
|
||||
cli.workers = Some(parse_worker_count(&value)?);
|
||||
Ok(None)
|
||||
}
|
||||
"--single-thread" => {
|
||||
/* As in the C CLI: zero workers plus a latch that suppresses the
|
||||
* automatic core-count resolution, so fileio runs its
|
||||
* single-thread streaming mode (slightly different from -T1). */
|
||||
cli.workers = Some(0);
|
||||
cli.single_thread = true;
|
||||
Ok(None)
|
||||
}
|
||||
"--memlimit" | "--memory" | "--memlimit-decompress" => {
|
||||
let value = next_value(attached, args, index, name)?;
|
||||
cli.mem_limit = Some(parse_u32(&value, "memory limit")?);
|
||||
@@ -868,6 +854,7 @@ fn parse_long_option(
|
||||
| "--trace"
|
||||
| "--format"
|
||||
| "--priority"
|
||||
| "--single-thread"
|
||||
| "--auto-threads"
|
||||
| "--fake-stdin-is-console"
|
||||
| "--fake-stdout-is-console"
|
||||
@@ -1025,7 +1012,7 @@ unsafe fn apply_preferences(cli: &Cli, prefs: *mut FIO_prefs_t, ctx: *mut FIO_ct
|
||||
}),
|
||||
);
|
||||
#[cfg(feature = "compression")]
|
||||
FIO_setNbWorkers(prefs, resolved_worker_count(cli.workers, cli.single_thread));
|
||||
FIO_setNbWorkers(prefs, resolved_worker_count(cli.workers));
|
||||
FIO_setLdmFlag(prefs, u32::from(cli.ldm));
|
||||
FIO_setAdaptiveMode(prefs, i32::from(cli.adapt));
|
||||
FIO_setRsyncable(prefs, i32::from(cli.rsyncable));
|
||||
@@ -1515,36 +1502,6 @@ mod tests {
|
||||
let cli = parse(&["zstdmt", "input"]);
|
||||
|
||||
assert_eq!(cli.workers, Some(0));
|
||||
assert!(!cli.single_thread);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_thread_pins_zero_workers() {
|
||||
let cli = parse(&["zstd", "--single-thread", "input"]);
|
||||
|
||||
assert_eq!(cli.workers, Some(0));
|
||||
assert!(cli.single_thread);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_later_thread_count_overrides_single_thread_workers() {
|
||||
/* Mirrors the C CLI: -T after --single-thread wins the worker count,
|
||||
* while the single-thread latch stays set. */
|
||||
let cli = parse(&["zstd", "--single-thread", "-T2", "input"]);
|
||||
|
||||
assert_eq!(cli.workers, Some(2));
|
||||
assert!(cli.single_thread);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_thread_rejects_attached_values() {
|
||||
let error = parse_args(vec![
|
||||
OsString::from("zstd"),
|
||||
OsString::from("--single-thread=1"),
|
||||
])
|
||||
.expect_err("an attached value must not activate --single-thread");
|
||||
|
||||
assert!(error.contains("does not take an argument"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -65,7 +65,7 @@ unsafe fn copy_bytes(dst: *mut u8, src: *const u8, size: usize) {
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn copy_huf_tables(dst: *mut ZSTD_hufCTables_t, src: *const ZSTD_hufCTables_t) {
|
||||
pub(crate) unsafe fn copy_huf_tables(dst: *mut ZSTD_hufCTables_t, src: *const ZSTD_hufCTables_t) {
|
||||
// Copy as bytes, not as a Rust struct: C's memcpy preserves the four tail
|
||||
// padding bytes that this layout has on 64-bit targets. Some C callers
|
||||
// copy or compare the complete entropy state, so leaving those bytes from
|
||||
@@ -192,8 +192,10 @@ fn min_literals_to_compress(strategy: c_int, huf_repeat: c_int) -> usize {
|
||||
}
|
||||
}
|
||||
|
||||
/// C's `ZSTD_minGain()`: minimum compression gain required to emit a
|
||||
/// compressed block or compressed literals section.
|
||||
#[inline]
|
||||
fn min_gain(src_size: usize, strategy: c_int) -> usize {
|
||||
pub(crate) fn min_gain(src_size: usize, strategy: c_int) -> usize {
|
||||
let min_log = if strategy >= ZSTD_BTULTRA {
|
||||
strategy.saturating_sub(1) as u32
|
||||
} else {
|
||||
|
||||
@@ -0,0 +1,1549 @@
|
||||
#![allow(non_camel_case_types)]
|
||||
#![allow(non_snake_case)]
|
||||
|
||||
//! Sequence statistics and seqStore entropy compression.
|
||||
//!
|
||||
//! This module translates the block entropy-statistics layer of
|
||||
//! `zstd_compress.c`: `ZSTD_seqToCodes()`, `ZSTD_buildSequencesStatistics()`,
|
||||
//! `ZSTD_entropyCompressSeqStore*()`, `ZSTD_buildBlockEntropyStats*()`, and
|
||||
//! `ZSTD_copyBlockSequences()`. The `ZSTD_CCtx` and `ZSTD_CCtx_params`
|
||||
//! layouts stay private to C: the C shims extract the sequence store, the
|
||||
//! entropy-table leaves, and the two parameter scalars these paths read
|
||||
//! (the compression strategy and the literals-compression switch). Block
|
||||
//! dispatch, block splitting, and the block-size estimation heuristics also
|
||||
//! remain in C for a later slice.
|
||||
|
||||
use crate::bits::ZSTD_highbit32;
|
||||
use crate::common::{
|
||||
DEFAULT_MAX_OFF, LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, LL_FSE_LOG, LONGNBSEQ, MAX_LL, MAX_ML,
|
||||
MAX_OFF, MAX_SEQ, MINMATCH, ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, ML_FSE_LOG, OFF_FSE_LOG,
|
||||
OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, ZSTD_MAX_FSE_HEADERS_SIZE, ZSTD_MAX_HUF_HEADER_SIZE,
|
||||
ZSTD_REP_NUM,
|
||||
};
|
||||
use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
|
||||
use crate::hist::{HIST_countFast_wksp, HIST_count_wksp};
|
||||
use crate::huf_compress::{
|
||||
HUF_buildCTable_wksp, HUF_estimateCompressedSize, HUF_optimalTableLog, HUF_validateCTable,
|
||||
HUF_writeCTable_wksp,
|
||||
};
|
||||
use crate::mem::{MEM_32bits, MEM_writeLE16};
|
||||
use crate::zstd_compress_literals::{
|
||||
copy_huf_tables, min_gain, ZSTD_compressLiterals, ZSTD_hufCTables_t,
|
||||
};
|
||||
use crate::zstd_compress_sequences::{
|
||||
SeqDef, ZSTD_buildCTable, ZSTD_encodeSequences, ZSTD_selectEncodingType,
|
||||
};
|
||||
use std::ffi::c_void;
|
||||
use std::mem::{size_of, size_of_val};
|
||||
use std::os::raw::{c_int, c_uint};
|
||||
use std::ptr;
|
||||
|
||||
const SET_BASIC: c_int = 0;
|
||||
const SET_RLE: c_int = 1;
|
||||
const SET_COMPRESSED: c_int = 2;
|
||||
const SET_REPEAT: c_int = 3;
|
||||
|
||||
const FSE_REPEAT_NONE: c_int = 0;
|
||||
|
||||
const HUF_REPEAT_NONE: c_int = 0;
|
||||
const HUF_REPEAT_CHECK: c_int = 1;
|
||||
const HUF_REPEAT_VALID: c_int = 2;
|
||||
|
||||
const HUF_FLAGS_OPTIMAL_DEPTH: c_int = 1 << 1;
|
||||
const HUF_OPTIMAL_DEPTH_THRESHOLD: c_int = 8; /* ZSTD_btultra */
|
||||
|
||||
const ZSTD_DEFAULT_DISALLOWED: c_int = 0;
|
||||
const ZSTD_DEFAULT_ALLOWED: c_int = 1;
|
||||
|
||||
const HUF_SYMBOLVALUE_MAX: c_uint = 255;
|
||||
const LIT_HUF_LOG: c_uint = 11;
|
||||
const COMPRESS_LITERALS_SIZE_MIN: usize = 63;
|
||||
const SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO: usize = 20;
|
||||
|
||||
const STREAM_ACCUMULATOR_MIN_32: u32 = 25;
|
||||
const STREAM_ACCUMULATOR_MIN_64: u32 = 57;
|
||||
|
||||
const LL_DELTA_CODE: u32 = 19;
|
||||
const ML_DELTA_CODE: u32 = 36;
|
||||
|
||||
pub(crate) const ZSTD_LLT_LITERAL_LENGTH: c_int = 1;
|
||||
pub(crate) const ZSTD_LLT_MATCH_LENGTH: c_int = 2;
|
||||
|
||||
/// `FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)` from the C headers.
|
||||
pub const OFF_CTABLE_SIZE: usize = 1 + (1 << (OFF_FSE_LOG - 1)) + ((MAX_OFF + 1) * 2);
|
||||
/// `FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)` from the C headers.
|
||||
pub const ML_CTABLE_SIZE: usize = 1 + (1 << (ML_FSE_LOG - 1)) + ((MAX_ML + 1) * 2);
|
||||
/// `FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)` from the C headers.
|
||||
pub const LL_CTABLE_SIZE: usize = 1 + (1 << (LL_FSE_LOG - 1)) + ((MAX_LL + 1) * 2);
|
||||
|
||||
/// ABI-compatible `SeqStore_t` leaf layout from `zstd_compress_internal.h`.
|
||||
///
|
||||
/// The C context itself remains opaque; its shims pass the `seqStore` member
|
||||
/// directly, so this small, stable hot-path structure is the only sequence
|
||||
/// storage representation crossing into Rust.
|
||||
#[repr(C)]
|
||||
pub struct SeqStore_t {
|
||||
pub sequencesStart: *mut SeqDef,
|
||||
pub sequences: *mut SeqDef,
|
||||
pub litStart: *mut u8,
|
||||
pub lit: *mut u8,
|
||||
pub llCode: *mut u8,
|
||||
pub mlCode: *mut u8,
|
||||
pub ofCode: *mut u8,
|
||||
pub maxNbSeq: usize,
|
||||
pub maxNbLit: usize,
|
||||
pub longLengthType: c_int,
|
||||
pub longLengthPos: u32,
|
||||
}
|
||||
|
||||
/// C's `ZSTD_fseCTables_t`. The table element type is `FSE_CTable`, an
|
||||
/// `unsigned`, and the table lengths are the header macros expanded above.
|
||||
#[repr(C)]
|
||||
pub struct ZSTD_fseCTables_t {
|
||||
pub offcodeCTable: [u32; OFF_CTABLE_SIZE],
|
||||
pub matchlengthCTable: [u32; ML_CTABLE_SIZE],
|
||||
pub litlengthCTable: [u32; LL_CTABLE_SIZE],
|
||||
pub offcode_repeatMode: c_int,
|
||||
pub matchlength_repeatMode: c_int,
|
||||
pub litlength_repeatMode: c_int,
|
||||
}
|
||||
|
||||
/// C's `ZSTD_entropyCTables_t`.
|
||||
#[repr(C)]
|
||||
pub struct ZSTD_entropyCTables_t {
|
||||
pub huf: ZSTD_hufCTables_t,
|
||||
pub fse: ZSTD_fseCTables_t,
|
||||
}
|
||||
|
||||
/// C's `ZSTD_compressedBlockState_t`.
|
||||
#[repr(C)]
|
||||
pub struct ZSTD_compressedBlockState_t {
|
||||
pub entropy: ZSTD_entropyCTables_t,
|
||||
pub rep: [u32; ZSTD_REP_NUM],
|
||||
}
|
||||
|
||||
/// C's `ZSTD_hufCTablesMetadata_t`.
|
||||
#[repr(C)]
|
||||
pub struct ZSTD_hufCTablesMetadata_t {
|
||||
pub hType: c_int,
|
||||
pub hufDesBuffer: [u8; ZSTD_MAX_HUF_HEADER_SIZE],
|
||||
pub hufDesSize: usize,
|
||||
}
|
||||
|
||||
/// C's `ZSTD_fseCTablesMetadata_t`.
|
||||
#[repr(C)]
|
||||
pub struct ZSTD_fseCTablesMetadata_t {
|
||||
pub llType: c_int,
|
||||
pub ofType: c_int,
|
||||
pub mlType: c_int,
|
||||
pub fseTablesBuffer: [u8; ZSTD_MAX_FSE_HEADERS_SIZE],
|
||||
pub fseTablesSize: usize,
|
||||
pub lastCountSize: usize,
|
||||
}
|
||||
|
||||
/// C's `ZSTD_entropyCTablesMetadata_t`.
|
||||
#[repr(C)]
|
||||
pub struct ZSTD_entropyCTablesMetadata_t {
|
||||
pub hufMetadata: ZSTD_hufCTablesMetadata_t,
|
||||
pub fseMetadata: ZSTD_fseCTablesMetadata_t,
|
||||
}
|
||||
|
||||
/// Public `ZSTD_Sequence` from `zstd.h`.
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct ZSTD_Sequence {
|
||||
pub offset: u32,
|
||||
pub litLength: u32,
|
||||
pub matchLength: u32,
|
||||
pub rep: u32,
|
||||
}
|
||||
|
||||
/// C's `SeqCollector` leaf from `zstd_compress_internal.h`.
|
||||
#[repr(C)]
|
||||
pub struct SeqCollector {
|
||||
pub collectSequences: c_int,
|
||||
pub seqStart: *mut ZSTD_Sequence,
|
||||
pub seqIndex: usize,
|
||||
pub maxSequences: usize,
|
||||
}
|
||||
|
||||
/// Result of `build_sequences_statistics()`; mirrors C's
|
||||
/// `ZSTD_symbolEncodingTypeStats_t`, which no longer crosses the boundary.
|
||||
pub(crate) struct SymbolEncodingTypeStats {
|
||||
pub ll_type: u32,
|
||||
pub off_type: u32,
|
||||
pub ml_type: u32,
|
||||
pub size: usize,
|
||||
/// Accounts for a bug in zstd 1.3.4; see
|
||||
/// `entropy_compress_seq_store_internal()`.
|
||||
pub last_count_size: usize,
|
||||
pub long_offsets: c_int,
|
||||
}
|
||||
|
||||
/// C's `ZSTD_updateRep()` repeat-offset history update.
|
||||
#[inline]
|
||||
pub(crate) fn update_rep(reps: &mut [u32; 3], off_base: u32, literal_length_is_zero: bool) {
|
||||
if off_base > 3 {
|
||||
reps[2] = reps[1];
|
||||
reps[1] = reps[0];
|
||||
reps[0] = off_base - 3;
|
||||
return;
|
||||
}
|
||||
let rep_code = off_base - 1 + u32::from(literal_length_is_zero);
|
||||
if rep_code == 0 {
|
||||
return;
|
||||
}
|
||||
let current_offset = if rep_code == 3 {
|
||||
reps[0].wrapping_sub(1)
|
||||
} else {
|
||||
reps[rep_code as usize]
|
||||
};
|
||||
reps[2] = if rep_code >= 2 { reps[1] } else { reps[2] };
|
||||
reps[1] = reps[0];
|
||||
reps[0] = current_offset;
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn ll_code(lit_length: u32) -> u32 {
|
||||
const LL_CODE: [u8; 64] = [
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20,
|
||||
20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 23, 23, 23,
|
||||
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
|
||||
];
|
||||
if lit_length > 63 {
|
||||
ZSTD_highbit32(lit_length) + LL_DELTA_CODE
|
||||
} else {
|
||||
LL_CODE[lit_length as usize] as u32
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn ml_code(ml_base: u32) -> u32 {
|
||||
const ML_CODE: [u8; 128] = [
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
|
||||
25, 26, 27, 28, 29, 30, 31, 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
|
||||
38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40,
|
||||
40, 40, 40, 40, 40, 40, 40, 40, 40, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
|
||||
41, 41, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
|
||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
|
||||
];
|
||||
if ml_base > 127 {
|
||||
ZSTD_highbit32(ml_base) + ML_DELTA_CODE
|
||||
} else {
|
||||
ML_CODE[ml_base as usize] as u32
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn copy_fse_tables(dst: *mut ZSTD_fseCTables_t, src: *const ZSTD_fseCTables_t) {
|
||||
unsafe {
|
||||
ptr::copy_nonoverlapping(
|
||||
src.cast::<u8>(),
|
||||
dst.cast::<u8>(),
|
||||
size_of::<ZSTD_fseCTables_t>(),
|
||||
)
|
||||
};
|
||||
}
|
||||
|
||||
/// Converts the stored lengths and distances into symbol codes.
|
||||
///
|
||||
/// Retains its original exported name: the C dictionary builder and the
|
||||
/// decodecorpus test tool link against it directly.
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_seqToCodes(seq_store: *const SeqStore_t) -> c_int {
|
||||
let store = unsafe { &*seq_store };
|
||||
let sequences = store.sequencesStart;
|
||||
let nb_seq = unsafe { store.sequences.offset_from(sequences) } as usize;
|
||||
let mut long_offsets = 0;
|
||||
debug_assert!(nb_seq <= store.maxNbSeq);
|
||||
|
||||
for index in 0..nb_seq {
|
||||
let sequence = unsafe { sequences.add(index).read() };
|
||||
let of_code = ZSTD_highbit32(sequence.offBase);
|
||||
unsafe {
|
||||
*store.llCode.add(index) = ll_code(sequence.litLength as u32) as u8;
|
||||
*store.ofCode.add(index) = of_code as u8;
|
||||
*store.mlCode.add(index) = ml_code(sequence.mlBase as u32) as u8;
|
||||
}
|
||||
debug_assert!(MEM_32bits() || of_code < STREAM_ACCUMULATOR_MIN_64);
|
||||
if MEM_32bits() && of_code >= STREAM_ACCUMULATOR_MIN_32 {
|
||||
long_offsets = 1;
|
||||
}
|
||||
}
|
||||
if store.longLengthType == ZSTD_LLT_LITERAL_LENGTH {
|
||||
unsafe { *store.llCode.add(store.longLengthPos as usize) = MAX_LL as u8 };
|
||||
}
|
||||
if store.longLengthType == ZSTD_LLT_MATCH_LENGTH {
|
||||
unsafe { *store.mlCode.add(store.longLengthPos as usize) = MAX_ML as u8 };
|
||||
}
|
||||
long_offsets
|
||||
}
|
||||
|
||||
/// C's `ZSTD_buildSequencesStatistics()`.
|
||||
///
|
||||
/// Returns the statistics with a zstd error code in the `size` field on
|
||||
/// failure, and updates `next_entropy` as a side effect. `nb_seq` must be
|
||||
/// greater than zero, and the entropy workspace must hold at least
|
||||
/// `ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1) * sizeof(U32)` bytes.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) unsafe fn build_sequences_statistics(
|
||||
seq_store: *const SeqStore_t,
|
||||
nb_seq: usize,
|
||||
prev_entropy: *const ZSTD_fseCTables_t,
|
||||
next_entropy: *mut ZSTD_fseCTables_t,
|
||||
dst: *mut u8,
|
||||
dst_capacity: usize,
|
||||
strategy: c_int,
|
||||
count_workspace: *mut c_uint,
|
||||
entropy_workspace: *mut c_void,
|
||||
entropy_wksp_size: usize,
|
||||
) -> SymbolEncodingTypeStats {
|
||||
let store = unsafe { &*seq_store };
|
||||
let mut stats = SymbolEncodingTypeStats {
|
||||
ll_type: 0,
|
||||
off_type: 0,
|
||||
ml_type: 0,
|
||||
size: 0,
|
||||
last_count_size: 0,
|
||||
long_offsets: 0,
|
||||
};
|
||||
let mut written = 0usize;
|
||||
|
||||
/* convert length/distances into codes */
|
||||
stats.long_offsets = unsafe { ZSTD_seqToCodes(seq_store) };
|
||||
debug_assert!(nb_seq != 0); /* ZSTD_selectEncodingType() divides by nb_seq */
|
||||
|
||||
/* build CTable for Literal Lengths */
|
||||
{
|
||||
let mut max = MAX_LL as c_uint;
|
||||
let most_frequent = unsafe {
|
||||
HIST_countFast_wksp(
|
||||
count_workspace,
|
||||
&mut max,
|
||||
store.llCode.cast::<c_void>(),
|
||||
nb_seq,
|
||||
entropy_workspace,
|
||||
entropy_wksp_size,
|
||||
)
|
||||
}; /* can't fail */
|
||||
unsafe { (*next_entropy).litlength_repeatMode = (*prev_entropy).litlength_repeatMode };
|
||||
stats.ll_type = unsafe {
|
||||
ZSTD_selectEncodingType(
|
||||
ptr::addr_of_mut!((*next_entropy).litlength_repeatMode),
|
||||
count_workspace,
|
||||
max,
|
||||
most_frequent,
|
||||
nb_seq,
|
||||
LL_FSE_LOG as c_uint,
|
||||
ptr::addr_of!((*prev_entropy).litlengthCTable).cast::<u32>(),
|
||||
LL_DEFAULT_NORM.as_ptr(),
|
||||
LL_DEFAULT_NORM_LOG,
|
||||
ZSTD_DEFAULT_ALLOWED,
|
||||
strategy,
|
||||
)
|
||||
} as u32;
|
||||
let count_size = unsafe {
|
||||
ZSTD_buildCTable(
|
||||
dst.add(written).cast::<c_void>(),
|
||||
dst_capacity - written,
|
||||
ptr::addr_of_mut!((*next_entropy).litlengthCTable).cast::<u32>(),
|
||||
LL_FSE_LOG as u32,
|
||||
stats.ll_type as c_int,
|
||||
count_workspace,
|
||||
max,
|
||||
store.llCode,
|
||||
nb_seq,
|
||||
LL_DEFAULT_NORM.as_ptr(),
|
||||
LL_DEFAULT_NORM_LOG,
|
||||
MAX_LL as u32,
|
||||
ptr::addr_of!((*prev_entropy).litlengthCTable).cast::<u32>(),
|
||||
size_of::<[u32; LL_CTABLE_SIZE]>(),
|
||||
entropy_workspace,
|
||||
entropy_wksp_size,
|
||||
)
|
||||
};
|
||||
if ERR_isError(count_size) {
|
||||
stats.size = count_size;
|
||||
return stats;
|
||||
}
|
||||
if stats.ll_type == SET_COMPRESSED as u32 {
|
||||
stats.last_count_size = count_size;
|
||||
}
|
||||
written += count_size;
|
||||
}
|
||||
/* build CTable for Offsets */
|
||||
{
|
||||
let mut max = MAX_OFF as c_uint;
|
||||
let most_frequent = unsafe {
|
||||
HIST_countFast_wksp(
|
||||
count_workspace,
|
||||
&mut max,
|
||||
store.ofCode.cast::<c_void>(),
|
||||
nb_seq,
|
||||
entropy_workspace,
|
||||
entropy_wksp_size,
|
||||
)
|
||||
}; /* can't fail */
|
||||
/* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */
|
||||
let default_policy = if max <= DEFAULT_MAX_OFF as c_uint {
|
||||
ZSTD_DEFAULT_ALLOWED
|
||||
} else {
|
||||
ZSTD_DEFAULT_DISALLOWED
|
||||
};
|
||||
unsafe { (*next_entropy).offcode_repeatMode = (*prev_entropy).offcode_repeatMode };
|
||||
stats.off_type = unsafe {
|
||||
ZSTD_selectEncodingType(
|
||||
ptr::addr_of_mut!((*next_entropy).offcode_repeatMode),
|
||||
count_workspace,
|
||||
max,
|
||||
most_frequent,
|
||||
nb_seq,
|
||||
OFF_FSE_LOG as c_uint,
|
||||
ptr::addr_of!((*prev_entropy).offcodeCTable).cast::<u32>(),
|
||||
OF_DEFAULT_NORM.as_ptr(),
|
||||
OF_DEFAULT_NORM_LOG,
|
||||
default_policy,
|
||||
strategy,
|
||||
)
|
||||
} as u32;
|
||||
let count_size = unsafe {
|
||||
ZSTD_buildCTable(
|
||||
dst.add(written).cast::<c_void>(),
|
||||
dst_capacity - written,
|
||||
ptr::addr_of_mut!((*next_entropy).offcodeCTable).cast::<u32>(),
|
||||
OFF_FSE_LOG as u32,
|
||||
stats.off_type as c_int,
|
||||
count_workspace,
|
||||
max,
|
||||
store.ofCode,
|
||||
nb_seq,
|
||||
OF_DEFAULT_NORM.as_ptr(),
|
||||
OF_DEFAULT_NORM_LOG,
|
||||
DEFAULT_MAX_OFF as u32,
|
||||
ptr::addr_of!((*prev_entropy).offcodeCTable).cast::<u32>(),
|
||||
size_of::<[u32; OFF_CTABLE_SIZE]>(),
|
||||
entropy_workspace,
|
||||
entropy_wksp_size,
|
||||
)
|
||||
};
|
||||
if ERR_isError(count_size) {
|
||||
stats.size = count_size;
|
||||
return stats;
|
||||
}
|
||||
if stats.off_type == SET_COMPRESSED as u32 {
|
||||
stats.last_count_size = count_size;
|
||||
}
|
||||
written += count_size;
|
||||
}
|
||||
/* build CTable for MatchLengths */
|
||||
{
|
||||
let mut max = MAX_ML as c_uint;
|
||||
let most_frequent = unsafe {
|
||||
HIST_countFast_wksp(
|
||||
count_workspace,
|
||||
&mut max,
|
||||
store.mlCode.cast::<c_void>(),
|
||||
nb_seq,
|
||||
entropy_workspace,
|
||||
entropy_wksp_size,
|
||||
)
|
||||
}; /* can't fail */
|
||||
unsafe { (*next_entropy).matchlength_repeatMode = (*prev_entropy).matchlength_repeatMode };
|
||||
stats.ml_type = unsafe {
|
||||
ZSTD_selectEncodingType(
|
||||
ptr::addr_of_mut!((*next_entropy).matchlength_repeatMode),
|
||||
count_workspace,
|
||||
max,
|
||||
most_frequent,
|
||||
nb_seq,
|
||||
ML_FSE_LOG as c_uint,
|
||||
ptr::addr_of!((*prev_entropy).matchlengthCTable).cast::<u32>(),
|
||||
ML_DEFAULT_NORM.as_ptr(),
|
||||
ML_DEFAULT_NORM_LOG,
|
||||
ZSTD_DEFAULT_ALLOWED,
|
||||
strategy,
|
||||
)
|
||||
} as u32;
|
||||
let count_size = unsafe {
|
||||
ZSTD_buildCTable(
|
||||
dst.add(written).cast::<c_void>(),
|
||||
dst_capacity - written,
|
||||
ptr::addr_of_mut!((*next_entropy).matchlengthCTable).cast::<u32>(),
|
||||
ML_FSE_LOG as u32,
|
||||
stats.ml_type as c_int,
|
||||
count_workspace,
|
||||
max,
|
||||
store.mlCode,
|
||||
nb_seq,
|
||||
ML_DEFAULT_NORM.as_ptr(),
|
||||
ML_DEFAULT_NORM_LOG,
|
||||
MAX_ML as u32,
|
||||
ptr::addr_of!((*prev_entropy).matchlengthCTable).cast::<u32>(),
|
||||
size_of::<[u32; ML_CTABLE_SIZE]>(),
|
||||
entropy_workspace,
|
||||
entropy_wksp_size,
|
||||
)
|
||||
};
|
||||
if ERR_isError(count_size) {
|
||||
stats.size = count_size;
|
||||
return stats;
|
||||
}
|
||||
if stats.ml_type == SET_COMPRESSED as u32 {
|
||||
stats.last_count_size = count_size;
|
||||
}
|
||||
written += count_size;
|
||||
}
|
||||
stats.size = written;
|
||||
stats
|
||||
}
|
||||
|
||||
/// C's `ZSTD_entropyCompressSeqStore_internal()`: compresses literals and
|
||||
/// sequences into one compressed-block body.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
unsafe fn entropy_compress_seq_store_internal(
|
||||
dst: *mut c_void,
|
||||
dst_capacity: usize,
|
||||
literals: *const c_void,
|
||||
lit_size: usize,
|
||||
seq_store: *const SeqStore_t,
|
||||
prev_entropy: *const ZSTD_entropyCTables_t,
|
||||
next_entropy: *mut ZSTD_entropyCTables_t,
|
||||
strategy: c_int,
|
||||
disable_literal_compression: c_int,
|
||||
entropy_workspace: *mut c_void,
|
||||
entropy_wksp_size: usize,
|
||||
bmi2: c_int,
|
||||
) -> usize {
|
||||
let count = entropy_workspace.cast::<c_uint>();
|
||||
let entropy_workspace = unsafe { count.add(MAX_SEQ + 1) }.cast::<c_void>();
|
||||
let entropy_wksp_size = entropy_wksp_size.wrapping_sub((MAX_SEQ + 1) * size_of::<c_uint>());
|
||||
let store = unsafe { &*seq_store };
|
||||
let sequences = store.sequencesStart;
|
||||
let nb_seq = unsafe { store.sequences.offset_from(sequences) } as usize;
|
||||
let ostart = dst.cast::<u8>();
|
||||
let mut written = 0usize;
|
||||
let last_count_size;
|
||||
let long_offsets;
|
||||
|
||||
/* Compress literals */
|
||||
{
|
||||
/* Base suspicion of uncompressibility on ratio of literals to sequences */
|
||||
let suspect_uncompressible =
|
||||
c_int::from(nb_seq == 0 || lit_size / nb_seq >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO);
|
||||
let c_size = unsafe {
|
||||
ZSTD_compressLiterals(
|
||||
dst,
|
||||
dst_capacity,
|
||||
literals,
|
||||
lit_size,
|
||||
entropy_workspace,
|
||||
entropy_wksp_size,
|
||||
ptr::addr_of!((*prev_entropy).huf),
|
||||
ptr::addr_of_mut!((*next_entropy).huf),
|
||||
strategy,
|
||||
disable_literal_compression,
|
||||
suspect_uncompressible,
|
||||
bmi2,
|
||||
)
|
||||
};
|
||||
if ERR_isError(c_size) {
|
||||
return c_size;
|
||||
}
|
||||
debug_assert!(c_size <= dst_capacity);
|
||||
written += c_size;
|
||||
}
|
||||
|
||||
/* Sequences Header */
|
||||
if (dst_capacity as isize).wrapping_sub(written as isize) < 3 /* max nbSeq Size */ + 1
|
||||
/* seqHead */
|
||||
{
|
||||
return ERROR(ZstdErrorCode::DstSizeTooSmall);
|
||||
}
|
||||
unsafe {
|
||||
if nb_seq < 128 {
|
||||
*ostart.add(written) = nb_seq as u8;
|
||||
written += 1;
|
||||
} else if nb_seq < LONGNBSEQ as usize {
|
||||
*ostart.add(written) = ((nb_seq >> 8) + 0x80) as u8;
|
||||
*ostart.add(written + 1) = nb_seq as u8;
|
||||
written += 2;
|
||||
} else {
|
||||
*ostart.add(written) = 0xFF;
|
||||
MEM_writeLE16(
|
||||
ostart.add(written + 1).cast::<c_void>(),
|
||||
(nb_seq - LONGNBSEQ as usize) as u16,
|
||||
);
|
||||
written += 3;
|
||||
}
|
||||
}
|
||||
if nb_seq == 0 {
|
||||
/* Copy the old tables over as if we repeated them */
|
||||
unsafe {
|
||||
copy_fse_tables(
|
||||
ptr::addr_of_mut!((*next_entropy).fse),
|
||||
ptr::addr_of!((*prev_entropy).fse),
|
||||
)
|
||||
};
|
||||
return written;
|
||||
}
|
||||
{
|
||||
let seq_head = written;
|
||||
written += 1;
|
||||
/* build stats for sequences */
|
||||
let stats = unsafe {
|
||||
build_sequences_statistics(
|
||||
seq_store,
|
||||
nb_seq,
|
||||
ptr::addr_of!((*prev_entropy).fse),
|
||||
ptr::addr_of_mut!((*next_entropy).fse),
|
||||
ostart.add(written),
|
||||
dst_capacity - written,
|
||||
strategy,
|
||||
count,
|
||||
entropy_workspace,
|
||||
entropy_wksp_size,
|
||||
)
|
||||
};
|
||||
if ERR_isError(stats.size) {
|
||||
return stats.size;
|
||||
}
|
||||
unsafe {
|
||||
*ostart.add(seq_head) =
|
||||
((stats.ll_type << 6) + (stats.off_type << 4) + (stats.ml_type << 2)) as u8;
|
||||
}
|
||||
last_count_size = stats.last_count_size;
|
||||
written += stats.size;
|
||||
long_offsets = stats.long_offsets;
|
||||
}
|
||||
|
||||
{
|
||||
let bitstream_size = unsafe {
|
||||
ZSTD_encodeSequences(
|
||||
ostart.add(written).cast::<c_void>(),
|
||||
dst_capacity - written,
|
||||
ptr::addr_of!((*next_entropy).fse.matchlengthCTable).cast::<u32>(),
|
||||
store.mlCode,
|
||||
ptr::addr_of!((*next_entropy).fse.offcodeCTable).cast::<u32>(),
|
||||
store.ofCode,
|
||||
ptr::addr_of!((*next_entropy).fse.litlengthCTable).cast::<u32>(),
|
||||
store.llCode,
|
||||
sequences,
|
||||
nb_seq,
|
||||
long_offsets,
|
||||
bmi2,
|
||||
)
|
||||
};
|
||||
if ERR_isError(bitstream_size) {
|
||||
return bitstream_size;
|
||||
}
|
||||
written += bitstream_size;
|
||||
/* zstd versions <= 1.3.4 mistakenly report corruption when
|
||||
* FSE_readNCount() receives a buffer < 4 bytes.
|
||||
* Fixed by https://github.com/facebook/zstd/pull/1146.
|
||||
* This can happen when the last set_compressed table present is 2
|
||||
* bytes and the bitstream is only one byte.
|
||||
* In this exceedingly rare case, we will simply emit an uncompressed
|
||||
* block, since it isn't worth optimizing.
|
||||
*/
|
||||
if last_count_size != 0 && last_count_size + bitstream_size < 4 {
|
||||
/* last_count_size >= 2 && bitstream_size > 0 ==> last_count_size == 3 */
|
||||
debug_assert!(last_count_size + bitstream_size == 3);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
written
|
||||
}
|
||||
|
||||
/// C's `ZSTD_entropyCompressSeqStore_wExtLitBuffer()`: adds the
|
||||
/// incompressibility fallbacks around the internal entropy pass.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
unsafe fn entropy_compress_seq_store_w_ext_lit_buffer(
|
||||
dst: *mut c_void,
|
||||
dst_capacity: usize,
|
||||
literals: *const c_void,
|
||||
lit_size: usize,
|
||||
block_size: usize,
|
||||
seq_store: *const SeqStore_t,
|
||||
prev_entropy: *const ZSTD_entropyCTables_t,
|
||||
next_entropy: *mut ZSTD_entropyCTables_t,
|
||||
strategy: c_int,
|
||||
disable_literal_compression: c_int,
|
||||
entropy_workspace: *mut c_void,
|
||||
entropy_wksp_size: usize,
|
||||
bmi2: c_int,
|
||||
) -> usize {
|
||||
let c_size = unsafe {
|
||||
entropy_compress_seq_store_internal(
|
||||
dst,
|
||||
dst_capacity,
|
||||
literals,
|
||||
lit_size,
|
||||
seq_store,
|
||||
prev_entropy,
|
||||
next_entropy,
|
||||
strategy,
|
||||
disable_literal_compression,
|
||||
entropy_workspace,
|
||||
entropy_wksp_size,
|
||||
bmi2,
|
||||
)
|
||||
};
|
||||
if c_size == 0 {
|
||||
return 0;
|
||||
}
|
||||
/* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block.
|
||||
* Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block.
|
||||
*/
|
||||
if c_size == ERROR(ZstdErrorCode::DstSizeTooSmall) && block_size <= dst_capacity {
|
||||
return 0; /* block not compressed */
|
||||
}
|
||||
if ERR_isError(c_size) {
|
||||
return c_size;
|
||||
}
|
||||
|
||||
/* Check compressibility */
|
||||
let max_c_size = block_size.wrapping_sub(min_gain(block_size, strategy));
|
||||
if c_size >= max_c_size {
|
||||
return 0; /* block not compressed */
|
||||
}
|
||||
c_size
|
||||
}
|
||||
|
||||
/// C ABI wrapper for the external-literal-buffer entropy pass; the C shim
|
||||
/// keeps the `ZSTD_entropyCompressSeqStore_internal()` name and extracts the
|
||||
/// two `ZSTD_CCtx_params` scalars.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_rust_entropyCompressSeqStore_internal(
|
||||
dst: *mut c_void,
|
||||
dst_capacity: usize,
|
||||
literals: *const c_void,
|
||||
lit_size: usize,
|
||||
seq_store: *const SeqStore_t,
|
||||
prev_entropy: *const ZSTD_entropyCTables_t,
|
||||
next_entropy: *mut ZSTD_entropyCTables_t,
|
||||
strategy: c_int,
|
||||
disable_literal_compression: c_int,
|
||||
entropy_workspace: *mut c_void,
|
||||
entropy_wksp_size: usize,
|
||||
bmi2: c_int,
|
||||
) -> usize {
|
||||
unsafe {
|
||||
entropy_compress_seq_store_internal(
|
||||
dst,
|
||||
dst_capacity,
|
||||
literals,
|
||||
lit_size,
|
||||
seq_store,
|
||||
prev_entropy,
|
||||
next_entropy,
|
||||
strategy,
|
||||
disable_literal_compression,
|
||||
entropy_workspace,
|
||||
entropy_wksp_size,
|
||||
bmi2,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// C ABI implementation behind the `ZSTD_entropyCompressSeqStore()` shim.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_rust_entropyCompressSeqStore(
|
||||
seq_store: *const SeqStore_t,
|
||||
prev_entropy: *const ZSTD_entropyCTables_t,
|
||||
next_entropy: *mut ZSTD_entropyCTables_t,
|
||||
strategy: c_int,
|
||||
disable_literal_compression: c_int,
|
||||
dst: *mut c_void,
|
||||
dst_capacity: usize,
|
||||
src_size: usize,
|
||||
entropy_workspace: *mut c_void,
|
||||
entropy_wksp_size: usize,
|
||||
bmi2: c_int,
|
||||
) -> usize {
|
||||
let store = unsafe { &*seq_store };
|
||||
let lit_size = unsafe { store.lit.offset_from(store.litStart) } as usize;
|
||||
unsafe {
|
||||
entropy_compress_seq_store_w_ext_lit_buffer(
|
||||
dst,
|
||||
dst_capacity,
|
||||
store.litStart.cast::<c_void>(),
|
||||
lit_size,
|
||||
src_size,
|
||||
seq_store,
|
||||
prev_entropy,
|
||||
next_entropy,
|
||||
strategy,
|
||||
disable_literal_compression,
|
||||
entropy_workspace,
|
||||
entropy_wksp_size,
|
||||
bmi2,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// C's `ZSTD_buildBlockEntropyStats_literals()`.
|
||||
///
|
||||
/// Stores the literals block type and Huffman description table into
|
||||
/// `huf_metadata` and returns the description size, or an error code.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
unsafe fn build_block_entropy_stats_literals(
|
||||
src: *const c_void,
|
||||
src_size: usize,
|
||||
prev_huf: *const ZSTD_hufCTables_t,
|
||||
next_huf: *mut ZSTD_hufCTables_t,
|
||||
huf_metadata: *mut ZSTD_hufCTablesMetadata_t,
|
||||
literals_compression_is_disabled: c_int,
|
||||
workspace: *mut c_void,
|
||||
wksp_size: usize,
|
||||
huf_flags: c_int,
|
||||
) -> usize {
|
||||
let count_wksp = workspace.cast::<c_uint>();
|
||||
let count_wksp_size = (HUF_SYMBOLVALUE_MAX as usize + 1) * size_of::<c_uint>();
|
||||
let node_wksp = unsafe { workspace.cast::<u8>().add(count_wksp_size) }.cast::<c_void>();
|
||||
let node_wksp_size = wksp_size.wrapping_sub(count_wksp_size);
|
||||
let mut max_symbol_value = HUF_SYMBOLVALUE_MAX;
|
||||
let mut huff_log = LIT_HUF_LOG;
|
||||
let mut repeat = unsafe { (*prev_huf).repeatMode };
|
||||
|
||||
/* Prepare nextEntropy assuming reusing the existing table */
|
||||
unsafe { copy_huf_tables(next_huf, prev_huf) };
|
||||
|
||||
if literals_compression_is_disabled != 0 {
|
||||
unsafe { (*huf_metadata).hType = SET_BASIC };
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* small ? don't even attempt compression (speed opt) */
|
||||
{
|
||||
let min_lit_size = if repeat == HUF_REPEAT_VALID {
|
||||
6
|
||||
} else {
|
||||
COMPRESS_LITERALS_SIZE_MIN
|
||||
};
|
||||
if src_size <= min_lit_size {
|
||||
unsafe { (*huf_metadata).hType = SET_BASIC };
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* Scan input and build symbol stats */
|
||||
{
|
||||
let largest = unsafe {
|
||||
HIST_count_wksp(
|
||||
count_wksp,
|
||||
&mut max_symbol_value,
|
||||
src,
|
||||
src_size,
|
||||
workspace,
|
||||
wksp_size,
|
||||
)
|
||||
};
|
||||
if ERR_isError(largest) {
|
||||
return largest;
|
||||
}
|
||||
if largest == src_size {
|
||||
/* only one literal symbol */
|
||||
unsafe { (*huf_metadata).hType = SET_RLE };
|
||||
return 0;
|
||||
}
|
||||
if largest <= (src_size >> 7) + 4 {
|
||||
/* heuristic: likely not compressible */
|
||||
unsafe { (*huf_metadata).hType = SET_BASIC };
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* Validate the previous Huffman table */
|
||||
if repeat == HUF_REPEAT_CHECK
|
||||
&& unsafe {
|
||||
HUF_validateCTable(
|
||||
ptr::addr_of!((*prev_huf).CTable).cast::<usize>(),
|
||||
count_wksp,
|
||||
max_symbol_value,
|
||||
)
|
||||
} == 0
|
||||
{
|
||||
repeat = HUF_REPEAT_NONE;
|
||||
}
|
||||
|
||||
/* Build Huffman Tree */
|
||||
unsafe {
|
||||
ptr::write_bytes(
|
||||
ptr::addr_of_mut!((*next_huf).CTable).cast::<u8>(),
|
||||
0,
|
||||
size_of_val(&(*next_huf).CTable),
|
||||
);
|
||||
}
|
||||
huff_log = unsafe {
|
||||
HUF_optimalTableLog(
|
||||
huff_log,
|
||||
src_size,
|
||||
max_symbol_value,
|
||||
node_wksp,
|
||||
node_wksp_size,
|
||||
ptr::addr_of_mut!((*next_huf).CTable).cast::<usize>(),
|
||||
count_wksp,
|
||||
huf_flags,
|
||||
)
|
||||
};
|
||||
debug_assert!(huff_log <= LIT_HUF_LOG);
|
||||
{
|
||||
let max_bits = unsafe {
|
||||
HUF_buildCTable_wksp(
|
||||
ptr::addr_of_mut!((*next_huf).CTable).cast::<usize>(),
|
||||
count_wksp,
|
||||
max_symbol_value,
|
||||
huff_log,
|
||||
node_wksp,
|
||||
node_wksp_size,
|
||||
)
|
||||
};
|
||||
if ERR_isError(max_bits) {
|
||||
return max_bits;
|
||||
}
|
||||
huff_log = max_bits as c_uint;
|
||||
}
|
||||
{
|
||||
/* Build and write the CTable */
|
||||
let new_c_size = unsafe {
|
||||
HUF_estimateCompressedSize(
|
||||
ptr::addr_of!((*next_huf).CTable).cast::<usize>(),
|
||||
count_wksp,
|
||||
max_symbol_value,
|
||||
)
|
||||
};
|
||||
let h_size = unsafe {
|
||||
HUF_writeCTable_wksp(
|
||||
ptr::addr_of_mut!((*huf_metadata).hufDesBuffer).cast::<c_void>(),
|
||||
size_of::<[u8; ZSTD_MAX_HUF_HEADER_SIZE]>(),
|
||||
ptr::addr_of!((*next_huf).CTable).cast::<usize>(),
|
||||
max_symbol_value,
|
||||
huff_log,
|
||||
node_wksp,
|
||||
node_wksp_size,
|
||||
)
|
||||
};
|
||||
/* Check against repeating the previous CTable */
|
||||
if repeat != HUF_REPEAT_NONE {
|
||||
let old_c_size = unsafe {
|
||||
HUF_estimateCompressedSize(
|
||||
ptr::addr_of!((*prev_huf).CTable).cast::<usize>(),
|
||||
count_wksp,
|
||||
max_symbol_value,
|
||||
)
|
||||
};
|
||||
if old_c_size < src_size
|
||||
&& (old_c_size <= h_size.wrapping_add(new_c_size)
|
||||
|| h_size.wrapping_add(12) >= src_size)
|
||||
{
|
||||
unsafe {
|
||||
copy_huf_tables(next_huf, prev_huf);
|
||||
(*huf_metadata).hType = SET_REPEAT;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
if new_c_size.wrapping_add(h_size) >= src_size {
|
||||
unsafe {
|
||||
copy_huf_tables(next_huf, prev_huf);
|
||||
(*huf_metadata).hType = SET_BASIC;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
unsafe {
|
||||
(*huf_metadata).hType = SET_COMPRESSED;
|
||||
(*next_huf).repeatMode = HUF_REPEAT_CHECK;
|
||||
}
|
||||
h_size
|
||||
}
|
||||
}
|
||||
|
||||
/// C's `ZSTD_buildDummySequencesStatistics()`: marks every encoding type as
|
||||
/// basic and resets the FSE repeat modes.
|
||||
unsafe fn build_dummy_sequences_statistics(
|
||||
next_entropy: *mut ZSTD_fseCTables_t,
|
||||
) -> SymbolEncodingTypeStats {
|
||||
unsafe {
|
||||
(*next_entropy).litlength_repeatMode = FSE_REPEAT_NONE;
|
||||
(*next_entropy).offcode_repeatMode = FSE_REPEAT_NONE;
|
||||
(*next_entropy).matchlength_repeatMode = FSE_REPEAT_NONE;
|
||||
}
|
||||
SymbolEncodingTypeStats {
|
||||
ll_type: SET_BASIC as u32,
|
||||
off_type: SET_BASIC as u32,
|
||||
ml_type: SET_BASIC as u32,
|
||||
size: 0,
|
||||
last_count_size: 0,
|
||||
long_offsets: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// C's `ZSTD_buildBlockEntropyStats_sequences()`.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
unsafe fn build_block_entropy_stats_sequences(
|
||||
seq_store: *const SeqStore_t,
|
||||
prev_entropy: *const ZSTD_fseCTables_t,
|
||||
next_entropy: *mut ZSTD_fseCTables_t,
|
||||
strategy: c_int,
|
||||
fse_metadata: *mut ZSTD_fseCTablesMetadata_t,
|
||||
workspace: *mut c_void,
|
||||
wksp_size: usize,
|
||||
) -> usize {
|
||||
let store = unsafe { &*seq_store };
|
||||
let nb_seq = unsafe { store.sequences.offset_from(store.sequencesStart) } as usize;
|
||||
let count_workspace = workspace.cast::<c_uint>();
|
||||
let entropy_workspace = unsafe { count_workspace.add(MAX_SEQ + 1) }.cast::<c_void>();
|
||||
let entropy_workspace_size = wksp_size.wrapping_sub((MAX_SEQ + 1) * size_of::<c_uint>());
|
||||
|
||||
let stats = if nb_seq != 0 {
|
||||
unsafe {
|
||||
build_sequences_statistics(
|
||||
seq_store,
|
||||
nb_seq,
|
||||
prev_entropy,
|
||||
next_entropy,
|
||||
ptr::addr_of_mut!((*fse_metadata).fseTablesBuffer).cast::<u8>(),
|
||||
ZSTD_MAX_FSE_HEADERS_SIZE,
|
||||
strategy,
|
||||
count_workspace,
|
||||
entropy_workspace,
|
||||
entropy_workspace_size,
|
||||
)
|
||||
}
|
||||
} else {
|
||||
unsafe { build_dummy_sequences_statistics(next_entropy) }
|
||||
};
|
||||
if ERR_isError(stats.size) {
|
||||
return stats.size;
|
||||
}
|
||||
unsafe {
|
||||
(*fse_metadata).llType = stats.ll_type as c_int;
|
||||
(*fse_metadata).ofType = stats.off_type as c_int;
|
||||
(*fse_metadata).mlType = stats.ml_type as c_int;
|
||||
(*fse_metadata).lastCountSize = stats.last_count_size;
|
||||
}
|
||||
stats.size
|
||||
}
|
||||
|
||||
/// C's `ZSTD_buildBlockEntropyStats()`: builds entropy statistics for one
|
||||
/// block. Requires an `ENTROPY_WORKSPACE_SIZE` workspace.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) unsafe fn build_block_entropy_stats(
|
||||
seq_store: *const SeqStore_t,
|
||||
prev_entropy: *const ZSTD_entropyCTables_t,
|
||||
next_entropy: *mut ZSTD_entropyCTables_t,
|
||||
strategy: c_int,
|
||||
disable_literal_compression: c_int,
|
||||
entropy_metadata: *mut ZSTD_entropyCTablesMetadata_t,
|
||||
workspace: *mut c_void,
|
||||
wksp_size: usize,
|
||||
) -> usize {
|
||||
let store = unsafe { &*seq_store };
|
||||
let lit_size = unsafe { store.lit.offset_from(store.litStart) } as usize;
|
||||
let huf_flags = if strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD {
|
||||
HUF_FLAGS_OPTIMAL_DEPTH
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
let huf_des_size = unsafe {
|
||||
build_block_entropy_stats_literals(
|
||||
store.litStart.cast::<c_void>(),
|
||||
lit_size,
|
||||
ptr::addr_of!((*prev_entropy).huf),
|
||||
ptr::addr_of_mut!((*next_entropy).huf),
|
||||
ptr::addr_of_mut!((*entropy_metadata).hufMetadata),
|
||||
disable_literal_compression,
|
||||
workspace,
|
||||
wksp_size,
|
||||
huf_flags,
|
||||
)
|
||||
};
|
||||
unsafe { (*entropy_metadata).hufMetadata.hufDesSize = huf_des_size };
|
||||
if ERR_isError(huf_des_size) {
|
||||
return huf_des_size;
|
||||
}
|
||||
|
||||
let fse_tables_size = unsafe {
|
||||
build_block_entropy_stats_sequences(
|
||||
seq_store,
|
||||
ptr::addr_of!((*prev_entropy).fse),
|
||||
ptr::addr_of_mut!((*next_entropy).fse),
|
||||
strategy,
|
||||
ptr::addr_of_mut!((*entropy_metadata).fseMetadata),
|
||||
workspace,
|
||||
wksp_size,
|
||||
)
|
||||
};
|
||||
unsafe { (*entropy_metadata).fseMetadata.fseTablesSize = fse_tables_size };
|
||||
if ERR_isError(fse_tables_size) {
|
||||
return fse_tables_size;
|
||||
}
|
||||
0
|
||||
}
|
||||
|
||||
/// C ABI wrapper behind the `ZSTD_buildBlockEntropyStats()` shim, which is
|
||||
/// still exported from C for the block splitter's size estimation.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_rust_buildBlockEntropyStats(
|
||||
seq_store: *const SeqStore_t,
|
||||
prev_entropy: *const ZSTD_entropyCTables_t,
|
||||
next_entropy: *mut ZSTD_entropyCTables_t,
|
||||
strategy: c_int,
|
||||
disable_literal_compression: c_int,
|
||||
entropy_metadata: *mut ZSTD_entropyCTablesMetadata_t,
|
||||
workspace: *mut c_void,
|
||||
wksp_size: usize,
|
||||
) -> usize {
|
||||
unsafe {
|
||||
build_block_entropy_stats(
|
||||
seq_store,
|
||||
prev_entropy,
|
||||
next_entropy,
|
||||
strategy,
|
||||
disable_literal_compression,
|
||||
entropy_metadata,
|
||||
workspace,
|
||||
wksp_size,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// C's `ZSTD_copyBlockSequences()`: exports a block's sequences in the public
|
||||
/// `ZSTD_Sequence` format, appending the last-literals block delimiter.
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_rust_copyBlockSequences(
|
||||
seq_collector: *mut SeqCollector,
|
||||
seq_store: *const SeqStore_t,
|
||||
prev_repcodes: *const u32,
|
||||
) -> usize {
|
||||
let collector = unsafe { &mut *seq_collector };
|
||||
let store = unsafe { &*seq_store };
|
||||
let in_seqs = store.sequencesStart;
|
||||
let nb_in_sequences = unsafe { store.sequences.offset_from(in_seqs) } as usize;
|
||||
let nb_in_literals = unsafe { store.lit.offset_from(store.litStart) } as usize;
|
||||
let out_seqs = if collector.seqIndex == 0 {
|
||||
collector.seqStart
|
||||
} else {
|
||||
unsafe { collector.seqStart.add(collector.seqIndex) }
|
||||
};
|
||||
let nb_out_sequences = nb_in_sequences + 1;
|
||||
let mut nb_out_literals = 0usize;
|
||||
|
||||
/* Bounds check that we have enough space for every input sequence
|
||||
* and the block delimiter
|
||||
*/
|
||||
debug_assert!(collector.seqIndex <= collector.maxSequences);
|
||||
if nb_out_sequences > collector.maxSequences.wrapping_sub(collector.seqIndex) {
|
||||
return ERROR(ZstdErrorCode::DstSizeTooSmall);
|
||||
}
|
||||
|
||||
let mut repcodes = [0u32; ZSTD_REP_NUM];
|
||||
unsafe { ptr::copy_nonoverlapping(prev_repcodes, repcodes.as_mut_ptr(), ZSTD_REP_NUM) };
|
||||
for index in 0..nb_in_sequences {
|
||||
let in_seq = unsafe { in_seqs.add(index).read() };
|
||||
let out = unsafe { out_seqs.add(index) };
|
||||
let mut lit_length = in_seq.litLength as u32;
|
||||
let mut match_length = in_seq.mlBase as u32 + MINMATCH as u32;
|
||||
let mut rep = 0u32;
|
||||
|
||||
/* Handle the possible single length >= 64K
|
||||
* There can only be one because we add MINMATCH to every match length,
|
||||
* and blocks are at most 128K.
|
||||
*/
|
||||
if index == store.longLengthPos as usize {
|
||||
if store.longLengthType == ZSTD_LLT_LITERAL_LENGTH {
|
||||
lit_length += 0x10000;
|
||||
} else if store.longLengthType == ZSTD_LLT_MATCH_LENGTH {
|
||||
match_length += 0x10000;
|
||||
}
|
||||
}
|
||||
|
||||
/* Determine the raw offset given the offBase, which may be a repcode. */
|
||||
let raw_offset = if (1..=ZSTD_REP_NUM as u32).contains(&in_seq.offBase) {
|
||||
let repcode = in_seq.offBase;
|
||||
debug_assert!(repcode > 0);
|
||||
rep = repcode;
|
||||
if lit_length != 0 {
|
||||
repcodes[repcode as usize - 1]
|
||||
} else if repcode == 3 {
|
||||
debug_assert!(repcodes[0] > 1);
|
||||
repcodes[0].wrapping_sub(1)
|
||||
} else {
|
||||
repcodes[repcode as usize]
|
||||
}
|
||||
} else {
|
||||
in_seq.offBase.wrapping_sub(ZSTD_REP_NUM as u32)
|
||||
};
|
||||
unsafe {
|
||||
(*out).litLength = lit_length;
|
||||
(*out).matchLength = match_length;
|
||||
(*out).offset = raw_offset;
|
||||
(*out).rep = rep;
|
||||
}
|
||||
|
||||
/* Update repcode history for the sequence */
|
||||
update_rep(&mut repcodes, in_seq.offBase, in_seq.litLength == 0);
|
||||
|
||||
nb_out_literals += lit_length as usize;
|
||||
}
|
||||
/* Insert last literals (if any exist) in the block as a sequence with ml == off == 0.
|
||||
* If there are no last literals, then we'll emit (of: 0, ml: 0, ll: 0), which is a marker
|
||||
* for the block boundary, according to the API.
|
||||
*/
|
||||
debug_assert!(nb_in_literals >= nb_out_literals);
|
||||
{
|
||||
let last_ll_size = nb_in_literals.wrapping_sub(nb_out_literals);
|
||||
unsafe {
|
||||
let last = out_seqs.add(nb_in_sequences);
|
||||
(*last).litLength = last_ll_size as u32;
|
||||
(*last).matchLength = 0;
|
||||
(*last).offset = 0;
|
||||
}
|
||||
}
|
||||
collector.seqIndex += nb_out_sequences;
|
||||
debug_assert!(collector.seqIndex <= collector.maxSequences);
|
||||
|
||||
0
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::mem::{align_of, offset_of};
|
||||
|
||||
const HUF_CTABLE_SIZE_ST: usize = HUF_SYMBOLVALUE_MAX as usize + 2;
|
||||
const ENTROPY_WORKSPACE_SIZE: usize = (8 << 10) + 512 + (MAX_SEQ + 2) * size_of::<u32>();
|
||||
|
||||
fn empty_huf_tables(repeat: c_int) -> ZSTD_hufCTables_t {
|
||||
ZSTD_hufCTables_t {
|
||||
CTable: [0; HUF_CTABLE_SIZE_ST],
|
||||
repeatMode: repeat,
|
||||
}
|
||||
}
|
||||
|
||||
fn empty_fse_tables(repeat: c_int) -> ZSTD_fseCTables_t {
|
||||
ZSTD_fseCTables_t {
|
||||
offcodeCTable: [0; OFF_CTABLE_SIZE],
|
||||
matchlengthCTable: [0; ML_CTABLE_SIZE],
|
||||
litlengthCTable: [0; LL_CTABLE_SIZE],
|
||||
offcode_repeatMode: repeat,
|
||||
matchlength_repeatMode: repeat,
|
||||
litlength_repeatMode: repeat,
|
||||
}
|
||||
}
|
||||
|
||||
struct TestSeqStore {
|
||||
sequences: Vec<SeqDef>,
|
||||
literals: Vec<u8>,
|
||||
ll_codes: Vec<u8>,
|
||||
ml_codes: Vec<u8>,
|
||||
of_codes: Vec<u8>,
|
||||
}
|
||||
|
||||
impl TestSeqStore {
|
||||
fn new(sequences: Vec<SeqDef>, literals: Vec<u8>) -> Self {
|
||||
let nb_seq = sequences.len();
|
||||
TestSeqStore {
|
||||
sequences,
|
||||
literals,
|
||||
ll_codes: vec![0; nb_seq],
|
||||
ml_codes: vec![0; nb_seq],
|
||||
of_codes: vec![0; nb_seq],
|
||||
}
|
||||
}
|
||||
|
||||
fn seq_store(&mut self) -> SeqStore_t {
|
||||
let nb_seq = self.sequences.len();
|
||||
let nb_lit = self.literals.len();
|
||||
SeqStore_t {
|
||||
sequencesStart: self.sequences.as_mut_ptr(),
|
||||
sequences: unsafe { self.sequences.as_mut_ptr().add(nb_seq) },
|
||||
litStart: self.literals.as_mut_ptr(),
|
||||
lit: unsafe { self.literals.as_mut_ptr().add(nb_lit) },
|
||||
llCode: self.ll_codes.as_mut_ptr(),
|
||||
mlCode: self.ml_codes.as_mut_ptr(),
|
||||
ofCode: self.of_codes.as_mut_ptr(),
|
||||
maxNbSeq: nb_seq,
|
||||
maxNbLit: nb_lit,
|
||||
longLengthType: 0,
|
||||
longLengthPos: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn c_leaf_layouts_match_supported_abis() {
|
||||
assert_eq!(size_of::<SeqDef>(), 8);
|
||||
assert_eq!(align_of::<SeqDef>(), align_of::<u32>());
|
||||
assert_eq!(offset_of!(SeqStore_t, sequencesStart), 0);
|
||||
assert_eq!(
|
||||
offset_of!(SeqStore_t, longLengthPos),
|
||||
9 * size_of::<usize>() + 4
|
||||
);
|
||||
assert_eq!(size_of::<SeqStore_t>(), 9 * size_of::<usize>() + 8);
|
||||
assert_eq!(size_of::<ZSTD_fseCTables_t>(), 3552);
|
||||
assert_eq!(offset_of!(ZSTD_compressedBlockState_t, entropy), 0);
|
||||
assert_eq!(
|
||||
offset_of!(ZSTD_compressedBlockState_t, rep),
|
||||
size_of::<ZSTD_entropyCTables_t>()
|
||||
);
|
||||
assert_eq!(size_of::<ZSTD_Sequence>(), 16);
|
||||
assert_eq!(offset_of!(SeqCollector, seqStart), size_of::<usize>());
|
||||
assert_eq!(offset_of!(SeqCollector, seqIndex), 2 * size_of::<usize>());
|
||||
assert_eq!(size_of::<SeqCollector>(), 4 * size_of::<usize>());
|
||||
if size_of::<usize>() == 8 {
|
||||
assert_eq!(size_of::<ZSTD_hufCTablesMetadata_t>(), 144);
|
||||
assert_eq!(size_of::<ZSTD_fseCTablesMetadata_t>(), 168);
|
||||
assert_eq!(size_of::<ZSTD_entropyCTablesMetadata_t>(), 312);
|
||||
assert_eq!(size_of::<ZSTD_entropyCTables_t>(), 5616);
|
||||
assert_eq!(size_of::<ZSTD_compressedBlockState_t>(), 5632);
|
||||
} else {
|
||||
assert_eq!(size_of::<ZSTD_hufCTablesMetadata_t>(), 136);
|
||||
assert_eq!(size_of::<ZSTD_fseCTablesMetadata_t>(), 156);
|
||||
assert_eq!(size_of::<ZSTD_entropyCTablesMetadata_t>(), 292);
|
||||
assert_eq!(size_of::<ZSTD_entropyCTables_t>(), 4584);
|
||||
assert_eq!(size_of::<ZSTD_compressedBlockState_t>(), 4596);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repcode_updates_match_the_c_sum_type_rules() {
|
||||
let mut reps = [1, 4, 8];
|
||||
update_rep(&mut reps, 10, false);
|
||||
assert_eq!(reps, [7, 1, 4]);
|
||||
update_rep(&mut reps, 1, true);
|
||||
assert_eq!(reps, [1, 7, 4]);
|
||||
update_rep(&mut reps, 2, false);
|
||||
assert_eq!(reps, [7, 1, 4]);
|
||||
update_rep(&mut reps, 3, false);
|
||||
assert_eq!(reps, [4, 7, 1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seq_to_codes_matches_the_reference_tables() {
|
||||
let mut store = TestSeqStore::new(
|
||||
vec![
|
||||
SeqDef {
|
||||
offBase: 4,
|
||||
litLength: 0,
|
||||
mlBase: 0,
|
||||
},
|
||||
SeqDef {
|
||||
offBase: 1,
|
||||
litLength: 63,
|
||||
mlBase: 127,
|
||||
},
|
||||
SeqDef {
|
||||
offBase: 1 << 20,
|
||||
litLength: 64,
|
||||
mlBase: 128,
|
||||
},
|
||||
],
|
||||
vec![0; 127],
|
||||
);
|
||||
let seq_store = store.seq_store();
|
||||
let long_offsets = unsafe { ZSTD_seqToCodes(&seq_store) };
|
||||
assert_eq!(long_offsets, 0);
|
||||
assert_eq!(store.ll_codes, [0, 24, 25]);
|
||||
assert_eq!(store.ml_codes, [0, 42, 43]);
|
||||
assert_eq!(store.of_codes, [2, 0, 20]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rle_sequences_produce_one_byte_table_headers() {
|
||||
// Five identical sequences select set_rle for all three symbol kinds;
|
||||
// each table header is then exactly the one RLE symbol byte.
|
||||
let mut store = TestSeqStore::new(
|
||||
vec![
|
||||
SeqDef {
|
||||
offBase: 8,
|
||||
litLength: 2,
|
||||
mlBase: 3,
|
||||
};
|
||||
5
|
||||
],
|
||||
vec![b'x'; 10],
|
||||
);
|
||||
let seq_store = store.seq_store();
|
||||
let prev = empty_fse_tables(FSE_REPEAT_NONE);
|
||||
let mut next = empty_fse_tables(FSE_REPEAT_NONE);
|
||||
let mut metadata =
|
||||
unsafe { std::mem::MaybeUninit::<ZSTD_fseCTablesMetadata_t>::zeroed().assume_init() };
|
||||
let mut workspace = vec![0u64; ENTROPY_WORKSPACE_SIZE / size_of::<u64>()];
|
||||
|
||||
let size = unsafe {
|
||||
build_block_entropy_stats_sequences(
|
||||
&seq_store,
|
||||
&prev,
|
||||
&mut next,
|
||||
1,
|
||||
&mut metadata,
|
||||
workspace.as_mut_ptr().cast::<c_void>(),
|
||||
ENTROPY_WORKSPACE_SIZE,
|
||||
)
|
||||
};
|
||||
assert_eq!(size, 3);
|
||||
assert_eq!(metadata.llType, SET_RLE);
|
||||
assert_eq!(metadata.ofType, SET_RLE);
|
||||
assert_eq!(metadata.mlType, SET_RLE);
|
||||
assert_eq!(metadata.lastCountSize, 0);
|
||||
assert_eq!(&metadata.fseTablesBuffer[..3], &[2, 3, 3]);
|
||||
assert_eq!(next.litlength_repeatMode, FSE_REPEAT_NONE);
|
||||
assert_eq!(next.offcode_repeatMode, FSE_REPEAT_NONE);
|
||||
assert_eq!(next.matchlength_repeatMode, FSE_REPEAT_NONE);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_seq_store_emits_raw_literals_and_repeats_tables() {
|
||||
let mut store = TestSeqStore::new(Vec::new(), b"hello".to_vec());
|
||||
let seq_store = store.seq_store();
|
||||
let prev = ZSTD_entropyCTables_t {
|
||||
huf: empty_huf_tables(HUF_REPEAT_NONE),
|
||||
fse: empty_fse_tables(2),
|
||||
};
|
||||
let mut next = ZSTD_entropyCTables_t {
|
||||
huf: empty_huf_tables(HUF_REPEAT_NONE),
|
||||
fse: empty_fse_tables(FSE_REPEAT_NONE),
|
||||
};
|
||||
let mut workspace = vec![0u64; ENTROPY_WORKSPACE_SIZE / size_of::<u64>()];
|
||||
let mut output = [0xA5u8; 32];
|
||||
|
||||
let size = unsafe {
|
||||
ZSTD_rust_entropyCompressSeqStore_internal(
|
||||
output.as_mut_ptr().cast::<c_void>(),
|
||||
output.len(),
|
||||
seq_store.litStart.cast::<c_void>(),
|
||||
5,
|
||||
&seq_store,
|
||||
&prev,
|
||||
&mut next,
|
||||
1,
|
||||
0,
|
||||
workspace.as_mut_ptr().cast::<c_void>(),
|
||||
ENTROPY_WORKSPACE_SIZE,
|
||||
0,
|
||||
)
|
||||
};
|
||||
// Raw literals header (5 << 3), five literal bytes, nbSeq == 0.
|
||||
assert_eq!(size, 7);
|
||||
assert_eq!(&output[..7], &[0x28, b'h', b'e', b'l', b'l', b'o', 0x00]);
|
||||
// With no sequences, the previous FSE tables are copied unchanged.
|
||||
assert_eq!(next.fse.offcode_repeatMode, 2);
|
||||
assert_eq!(next.fse.matchlength_repeatMode, 2);
|
||||
assert_eq!(next.fse.litlength_repeatMode, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn literal_stats_pick_rle_and_compressed_types() {
|
||||
let mut workspace = vec![0u64; ENTROPY_WORKSPACE_SIZE / size_of::<u64>()];
|
||||
let prev = empty_huf_tables(HUF_REPEAT_NONE);
|
||||
let mut next = empty_huf_tables(HUF_REPEAT_NONE);
|
||||
let mut metadata =
|
||||
unsafe { std::mem::MaybeUninit::<ZSTD_hufCTablesMetadata_t>::zeroed().assume_init() };
|
||||
|
||||
let rle = [7u8; 128];
|
||||
let size = unsafe {
|
||||
build_block_entropy_stats_literals(
|
||||
rle.as_ptr().cast::<c_void>(),
|
||||
rle.len(),
|
||||
&prev,
|
||||
&mut next,
|
||||
&mut metadata,
|
||||
0,
|
||||
workspace.as_mut_ptr().cast::<c_void>(),
|
||||
ENTROPY_WORKSPACE_SIZE,
|
||||
0,
|
||||
)
|
||||
};
|
||||
assert_eq!(size, 0);
|
||||
assert_eq!(metadata.hType, SET_RLE);
|
||||
|
||||
let mut compressible = [0u8; 1024];
|
||||
for (index, byte) in compressible.iter_mut().enumerate() {
|
||||
*byte = (index % 4) as u8;
|
||||
}
|
||||
let size = unsafe {
|
||||
build_block_entropy_stats_literals(
|
||||
compressible.as_ptr().cast::<c_void>(),
|
||||
compressible.len(),
|
||||
&prev,
|
||||
&mut next,
|
||||
&mut metadata,
|
||||
0,
|
||||
workspace.as_mut_ptr().cast::<c_void>(),
|
||||
ENTROPY_WORKSPACE_SIZE,
|
||||
0,
|
||||
)
|
||||
};
|
||||
assert!(!ERR_isError(size));
|
||||
assert!(size > 0);
|
||||
assert_eq!(metadata.hType, SET_COMPRESSED);
|
||||
assert_eq!(metadata.hufDesSize, 0); // caller stores the returned size
|
||||
assert_eq!(next.repeatMode, HUF_REPEAT_CHECK);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn copied_sequences_resolve_repcodes_and_append_the_delimiter() {
|
||||
let mut store = TestSeqStore::new(
|
||||
vec![
|
||||
SeqDef {
|
||||
offBase: 3 + 100, // raw offset 100
|
||||
litLength: 4,
|
||||
mlBase: 5,
|
||||
},
|
||||
SeqDef {
|
||||
offBase: 1, // repcode 1 with literals: rep[0] == 100
|
||||
litLength: 2,
|
||||
mlBase: 7,
|
||||
},
|
||||
SeqDef {
|
||||
offBase: 3, // repcode 3 without literals: rep[0] - 1 == 99
|
||||
litLength: 0,
|
||||
mlBase: 9,
|
||||
},
|
||||
],
|
||||
vec![0; 10],
|
||||
);
|
||||
let seq_store = store.seq_store();
|
||||
let mut out = [ZSTD_Sequence {
|
||||
offset: 0xAA,
|
||||
litLength: 0xAA,
|
||||
matchLength: 0xAA,
|
||||
rep: 0xAA,
|
||||
}; 8];
|
||||
let mut collector = SeqCollector {
|
||||
collectSequences: 1,
|
||||
seqStart: out.as_mut_ptr(),
|
||||
seqIndex: 1,
|
||||
maxSequences: 8,
|
||||
};
|
||||
let reps = [1u32, 4, 8];
|
||||
|
||||
let result =
|
||||
unsafe { ZSTD_rust_copyBlockSequences(&mut collector, &seq_store, reps.as_ptr()) };
|
||||
assert_eq!(result, 0);
|
||||
assert_eq!(collector.seqIndex, 5);
|
||||
let expect = [
|
||||
(100, 4, 5 + 3, 0),
|
||||
(100, 2, 7 + 3, 1),
|
||||
(99, 0, 9 + 3, 3),
|
||||
(0, 10 - 6, 0, 0xAA), // delimiter: last literals; rep untouched
|
||||
];
|
||||
for (index, &(offset, lit, matched, rep)) in expect.iter().enumerate() {
|
||||
assert_eq!(out[1 + index].offset, offset, "offset {index}");
|
||||
assert_eq!(out[1 + index].litLength, lit, "litLength {index}");
|
||||
assert_eq!(out[1 + index].matchLength, matched, "matchLength {index}");
|
||||
assert_eq!(out[1 + index].rep, rep, "rep {index}");
|
||||
}
|
||||
|
||||
collector.maxSequences = 5;
|
||||
let result =
|
||||
unsafe { ZSTD_rust_copyBlockSequences(&mut collector, &seq_store, reps.as_ptr()) };
|
||||
assert_eq!(result, ERROR(ZstdErrorCode::DstSizeTooSmall));
|
||||
}
|
||||
}
|
||||
@@ -6,13 +6,13 @@
|
||||
//! This is the Rust implementation of `zstd_compress_superblock.c`. Its C
|
||||
//! entry-point shim only extracts fields from the opaque `ZSTD_CCtx`; all
|
||||
//! sequence partitioning, literal and sequence section writing, entropy
|
||||
//! fallback, and repcode repair remain here. The leaf layouts below are
|
||||
//! intentionally kept C-shaped and checked for both supported pointer widths.
|
||||
//! fallback, and repcode repair remain here. The C-shaped leaf layouts are
|
||||
//! shared with `zstd_compress_stats`, which also owns the block entropy
|
||||
//! statistics this module consumes.
|
||||
|
||||
use crate::common::{
|
||||
LL_BITS, LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, MAX_LL, MAX_ML, MAX_OFF, MINMATCH, ML_BITS,
|
||||
ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG,
|
||||
ZSTD_MAX_FSE_HEADERS_SIZE, ZSTD_MAX_HUF_HEADER_SIZE,
|
||||
};
|
||||
use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
|
||||
use crate::hist::{HIST_countFast_wksp, HIST_count_wksp};
|
||||
@@ -23,9 +23,17 @@ use crate::mem::{MEM_32bits, MEM_writeLE16, MEM_writeLE24, MEM_writeLE32};
|
||||
use crate::zstd_compress_literals::{
|
||||
ZSTD_compressRleLiteralsBlock, ZSTD_hufCTables_t, ZSTD_noCompressLiterals,
|
||||
};
|
||||
use crate::zstd_compress_sequences::{
|
||||
SeqDef, ZSTD_crossEntropyCost, ZSTD_encodeSequences, ZSTD_fseBitCost,
|
||||
};
|
||||
use crate::zstd_compress_stats::{
|
||||
build_block_entropy_stats, update_rep, SeqStore_t, ZSTD_compressedBlockState_t,
|
||||
ZSTD_entropyCTablesMetadata_t, ZSTD_entropyCTables_t, ZSTD_fseCTablesMetadata_t,
|
||||
ZSTD_fseCTables_t, ZSTD_hufCTablesMetadata_t,
|
||||
};
|
||||
use std::ffi::c_void;
|
||||
use std::mem::MaybeUninit;
|
||||
use std::os::raw::{c_int, c_short, c_uint};
|
||||
use std::os::raw::c_int;
|
||||
use std::ptr;
|
||||
|
||||
const SET_BASIC: c_int = 0;
|
||||
@@ -42,121 +50,6 @@ const LONG_NB_SEQ: usize = 0x7f00;
|
||||
const DEFAULT_MAX_OFF: u32 = 28;
|
||||
const BYTE_SCALE: usize = 256;
|
||||
|
||||
const OFF_CTABLE_SIZE: usize = 1 + (1 << 7) + ((MAX_OFF + 1) * 2);
|
||||
const ML_CTABLE_SIZE: usize = 1 + (1 << 8) + ((MAX_ML + 1) * 2);
|
||||
const LL_CTABLE_SIZE: usize = 1 + (1 << 8) + ((MAX_LL + 1) * 2);
|
||||
|
||||
/// ABI-compatible `SeqDef` from `zstd_compress_internal.h`.
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
struct SeqDef {
|
||||
offBase: u32,
|
||||
litLength: u16,
|
||||
mlBase: u16,
|
||||
}
|
||||
|
||||
/// ABI-compatible `SeqStore_t` leaf layout.
|
||||
///
|
||||
/// The C context itself remains opaque. The C shim passes its `seqStore`
|
||||
/// member directly, so this small, stable hot-path structure is the only
|
||||
/// sequence storage representation crossing into Rust.
|
||||
#[repr(C)]
|
||||
struct SeqStore_t {
|
||||
sequencesStart: *mut SeqDef,
|
||||
sequences: *mut SeqDef,
|
||||
litStart: *mut u8,
|
||||
lit: *mut u8,
|
||||
llCode: *mut u8,
|
||||
mlCode: *mut u8,
|
||||
ofCode: *mut u8,
|
||||
maxNbSeq: usize,
|
||||
maxNbLit: usize,
|
||||
longLengthType: c_int,
|
||||
longLengthPos: u32,
|
||||
}
|
||||
|
||||
/// C's `ZSTD_fseCTables_t`. The table element type is `FSE_CTable`, an
|
||||
/// `unsigned`, and the table lengths are the header macros expanded above.
|
||||
#[repr(C)]
|
||||
struct ZSTD_fseCTables_t {
|
||||
offcodeCTable: [u32; OFF_CTABLE_SIZE],
|
||||
matchlengthCTable: [u32; ML_CTABLE_SIZE],
|
||||
litlengthCTable: [u32; LL_CTABLE_SIZE],
|
||||
offcode_repeatMode: c_int,
|
||||
matchlength_repeatMode: c_int,
|
||||
litlength_repeatMode: c_int,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
struct ZSTD_entropyCTables_t {
|
||||
huf: ZSTD_hufCTables_t,
|
||||
fse: ZSTD_fseCTables_t,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
struct ZSTD_compressedBlockState_t {
|
||||
entropy: ZSTD_entropyCTables_t,
|
||||
rep: [u32; 3],
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
struct ZSTD_hufCTablesMetadata_t {
|
||||
hType: c_int,
|
||||
hufDesBuffer: [u8; ZSTD_MAX_HUF_HEADER_SIZE],
|
||||
hufDesSize: usize,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
struct ZSTD_fseCTablesMetadata_t {
|
||||
llType: c_int,
|
||||
ofType: c_int,
|
||||
mlType: c_int,
|
||||
fseTablesBuffer: [u8; ZSTD_MAX_FSE_HEADERS_SIZE],
|
||||
fseTablesSize: usize,
|
||||
lastCountSize: usize,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
struct ZSTD_entropyCTablesMetadata_t {
|
||||
hufMetadata: ZSTD_hufCTablesMetadata_t,
|
||||
fseMetadata: ZSTD_fseCTablesMetadata_t,
|
||||
}
|
||||
|
||||
unsafe extern "C" {
|
||||
fn ZSTD_buildBlockEntropyStats(
|
||||
seq_store: *const SeqStore_t,
|
||||
prev_entropy: *const ZSTD_entropyCTables_t,
|
||||
next_entropy: *mut ZSTD_entropyCTables_t,
|
||||
cctx_params: *const c_void,
|
||||
entropy_metadata: *mut ZSTD_entropyCTablesMetadata_t,
|
||||
workspace: *mut c_void,
|
||||
wksp_size: usize,
|
||||
) -> usize;
|
||||
|
||||
fn ZSTD_encodeSequences(
|
||||
dst: *mut c_void,
|
||||
dst_capacity: usize,
|
||||
ctable_match_length: *const u32,
|
||||
ml_code_table: *const u8,
|
||||
ctable_offset_bits: *const u32,
|
||||
of_code_table: *const u8,
|
||||
ctable_lit_length: *const u32,
|
||||
ll_code_table: *const u8,
|
||||
sequences: *const SeqDef,
|
||||
nb_seq: usize,
|
||||
long_offsets: c_int,
|
||||
bmi2: c_int,
|
||||
) -> usize;
|
||||
|
||||
fn ZSTD_fseBitCost(ctable: *const u32, count: *const u32, max: c_uint) -> usize;
|
||||
fn ZSTD_crossEntropyCost(
|
||||
norm: *const c_short,
|
||||
accuracy_log: c_uint,
|
||||
count: *const c_uint,
|
||||
max: c_uint,
|
||||
) -> usize;
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn remaining_capacity(capacity: usize, written: usize) -> Result<usize, usize> {
|
||||
capacity
|
||||
@@ -838,28 +731,6 @@ unsafe fn size_block_sequences(
|
||||
nb_seq
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn update_rep(reps: &mut [u32; 3], off_base: u32, literal_length_is_zero: bool) {
|
||||
if off_base > 3 {
|
||||
reps[2] = reps[1];
|
||||
reps[1] = reps[0];
|
||||
reps[0] = off_base - 3;
|
||||
return;
|
||||
}
|
||||
let rep_code = off_base - 1 + u32::from(literal_length_is_zero);
|
||||
if rep_code == 0 {
|
||||
return;
|
||||
}
|
||||
let current_offset = if rep_code == 3 {
|
||||
reps[0].wrapping_sub(1)
|
||||
} else {
|
||||
reps[rep_code as usize]
|
||||
};
|
||||
reps[2] = if rep_code >= 2 { reps[1] } else { reps[2] };
|
||||
reps[1] = reps[0];
|
||||
reps[0] = current_offset;
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments, clippy::manual_checked_ops)]
|
||||
unsafe fn compress_subblock_multi(
|
||||
seq_store: *const SeqStore_t,
|
||||
@@ -1098,16 +969,18 @@ unsafe fn compress_subblock_multi(
|
||||
|
||||
/// C ABI implementation called by the declaration-only C superblock shim.
|
||||
///
|
||||
/// `cctx_params` deliberately remains opaque: only its two required scalar
|
||||
/// fields are read by the shim, while the existing C entropy builder receives
|
||||
/// the original pointer unchanged.
|
||||
/// `ZSTD_CCtx_params` deliberately remains opaque: the C shim extracts the
|
||||
/// two scalar fields this path reads (the strategy and the
|
||||
/// literals-compression switch) and the crate-internal entropy statistics
|
||||
/// builder receives them directly.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_rust_compressSuperBlock(
|
||||
seq_store: *const c_void,
|
||||
prev_cblock: *const c_void,
|
||||
next_cblock: *mut c_void,
|
||||
cctx_params: *const c_void,
|
||||
strategy: c_int,
|
||||
disable_literal_compression: c_int,
|
||||
workspace: *mut c_void,
|
||||
wksp_size: usize,
|
||||
bmi2: c_int,
|
||||
@@ -1122,16 +995,17 @@ pub unsafe extern "C" fn ZSTD_rust_compressSuperBlock(
|
||||
let seq_store = seq_store.cast::<SeqStore_t>();
|
||||
let prev_cblock = prev_cblock.cast::<ZSTD_compressedBlockState_t>();
|
||||
let next_cblock = next_cblock.cast::<ZSTD_compressedBlockState_t>();
|
||||
// C only writes the used prefixes of its two metadata byte buffers. Start
|
||||
// with initialized storage so treating the completed C struct as a Rust
|
||||
// value never exposes uninitialized array elements.
|
||||
// The statistics builder only writes the used prefixes of the two
|
||||
// metadata byte buffers. Start with initialized storage so treating the
|
||||
// completed struct as a Rust value never exposes uninitialized elements.
|
||||
let mut entropy_metadata = MaybeUninit::<ZSTD_entropyCTablesMetadata_t>::zeroed();
|
||||
let entropy_result = unsafe {
|
||||
ZSTD_buildBlockEntropyStats(
|
||||
build_block_entropy_stats(
|
||||
seq_store,
|
||||
ptr::addr_of!((*prev_cblock).entropy),
|
||||
ptr::addr_of_mut!((*next_cblock).entropy),
|
||||
cctx_params,
|
||||
strategy,
|
||||
disable_literal_compression,
|
||||
entropy_metadata.as_mut_ptr(),
|
||||
workspace,
|
||||
wksp_size,
|
||||
@@ -1163,50 +1037,34 @@ pub unsafe extern "C" fn ZSTD_rust_compressSuperBlock(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
// The shared C leaf layouts and the repcode update rules are asserted in
|
||||
// `zstd_compress_stats`, which owns those definitions.
|
||||
use super::*;
|
||||
use std::mem::{align_of, offset_of, size_of};
|
||||
|
||||
#[test]
|
||||
fn c_leaf_layouts_match_supported_abis() {
|
||||
assert_eq!(size_of::<SeqDef>(), 8);
|
||||
assert_eq!(align_of::<SeqDef>(), align_of::<u32>());
|
||||
assert_eq!(offset_of!(SeqStore_t, sequencesStart), 0);
|
||||
assert_eq!(
|
||||
offset_of!(SeqStore_t, longLengthPos),
|
||||
9 * size_of::<usize>() + 4
|
||||
);
|
||||
assert_eq!(size_of::<SeqStore_t>(), 9 * size_of::<usize>() + 8);
|
||||
assert_eq!(size_of::<ZSTD_fseCTables_t>(), 3552);
|
||||
assert_eq!(offset_of!(ZSTD_compressedBlockState_t, entropy), 0);
|
||||
assert_eq!(
|
||||
offset_of!(ZSTD_compressedBlockState_t, rep),
|
||||
size_of::<ZSTD_entropyCTables_t>()
|
||||
);
|
||||
if size_of::<usize>() == 8 {
|
||||
assert_eq!(size_of::<ZSTD_hufCTablesMetadata_t>(), 144);
|
||||
assert_eq!(size_of::<ZSTD_fseCTablesMetadata_t>(), 168);
|
||||
assert_eq!(size_of::<ZSTD_entropyCTablesMetadata_t>(), 312);
|
||||
assert_eq!(size_of::<ZSTD_entropyCTables_t>(), 5616);
|
||||
assert_eq!(size_of::<ZSTD_compressedBlockState_t>(), 5632);
|
||||
} else {
|
||||
assert_eq!(size_of::<ZSTD_hufCTablesMetadata_t>(), 136);
|
||||
assert_eq!(size_of::<ZSTD_fseCTablesMetadata_t>(), 156);
|
||||
assert_eq!(size_of::<ZSTD_entropyCTablesMetadata_t>(), 292);
|
||||
assert_eq!(size_of::<ZSTD_entropyCTables_t>(), 4584);
|
||||
assert_eq!(size_of::<ZSTD_compressedBlockState_t>(), 4596);
|
||||
}
|
||||
}
|
||||
fn subblock_sizing_respects_the_budget() {
|
||||
let sequences = [SeqDef {
|
||||
offBase: 4,
|
||||
litLength: 10,
|
||||
mlBase: 10,
|
||||
}; 8];
|
||||
|
||||
#[test]
|
||||
fn repcode_updates_match_the_c_sum_type_rules() {
|
||||
let mut reps = [1, 4, 8];
|
||||
update_rep(&mut reps, 10, false);
|
||||
assert_eq!(reps, [7, 1, 4]);
|
||||
update_rep(&mut reps, 1, true);
|
||||
assert_eq!(reps, [1, 7, 4]);
|
||||
update_rep(&mut reps, 2, false);
|
||||
assert_eq!(reps, [7, 1, 4]);
|
||||
update_rep(&mut reps, 3, false);
|
||||
assert_eq!(reps, [4, 7, 1]);
|
||||
// A generous budget keeps every sequence in one sub-block.
|
||||
let all = unsafe {
|
||||
size_block_sequences(
|
||||
sequences.as_ptr(),
|
||||
sequences.len(),
|
||||
usize::MAX,
|
||||
256,
|
||||
256,
|
||||
true,
|
||||
)
|
||||
};
|
||||
assert_eq!(all, sequences.len());
|
||||
|
||||
// The first sequence always survives, even over budget.
|
||||
let first =
|
||||
unsafe { size_block_sequences(sequences.as_ptr(), sequences.len(), 0, 256, 256, true) };
|
||||
assert_eq!(first, 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,8 +79,6 @@ RUST_TARGET_DIR := $(RUST_DIR)/target/$(RUST_BUILD_CONFIG)
|
||||
RUST_STATICLIB := $(RUST_TARGET_DIR)/release/libzstd_rs.a
|
||||
RUST_TARGET_32 ?= i686-unknown-linux-gnu
|
||||
RUST_STATICLIB_32 := $(RUST_TARGET_DIR)/$(RUST_TARGET_32)/release/libzstd_rs.a
|
||||
# Tests build every library module, so they use the crate's default feature
|
||||
# set (compression, decompression, and dict-builder) plus any forced HUF mode.
|
||||
RUST_CARGO_FLAGS := --manifest-path $(RUST_MANIFEST) --release \
|
||||
--target-dir $(RUST_TARGET_DIR)
|
||||
ifneq ($(RUST_HUF_FEATURE),)
|
||||
|
||||
Reference in New Issue
Block a user