feat(compress): move split-block emission into Rust
Move the post-split partition loop out of zstd_compress.c while keeping the private CCtx, matchfinder sequence-store construction, and split discovery in C. Rust now receives a layout-asserted projection containing sequence-store views, block-state slots, workspace and scalar policy, then mirrors the C loop's dRep/cRep histories, final-literal accounting, repeated single-block serialization, and final dRep publication. Remove the obsolete C sequence-store/count/chunk wrappers and the debug-only C size-estimation path that was coupled to the old loop. Add a focused Rust fixture covering partition payloads and final literals, and document the new ownership boundary for the split search and emission paths. Test Plan: - cargo test --manifest-path rust/Cargo.toml --lib -- --test-threads=1 (448 passed) - cargo clippy --manifest-path rust/Cargo.toml --lib -- -D warnings (passed) - cargo clippy --manifest-path rust/Cargo.toml -- -D warnings (passed) - cargo +nightly fmt --manifest-path rust/Cargo.toml -- --check (passed) - cargo clippy --tests/--benches remain blocked only by the pre-existing manual_repeat_n lint in one_shot_promotes_nonfirst_rle_blocks - make -B -C lib -j2 lib (passed) - make -B -C tests -j2 test-zstd (passed) - make -B -C tests -j2 test-cli-tests (41 passed) - ZSTREAM_TESTTIME=-T2s make -B -C tests -j2 test-zstream (passed) - FUZZERTEST=-T5s make -B -C tests -j2 test-fuzzer (252 passed) - make -B -C tests/fuzz -j2 all and sequence_compression_api (passed)
This commit is contained in:
+49
-219
@@ -120,46 +120,50 @@ typedef char ZSTD_rust_target_cblock_state_layout[
|
||||
&& sizeof(ZSTD_rust_targetCBlockSizeState)
|
||||
== (sizeof(void*) == 8 ? 72 : 44))
|
||||
? 1 : -1];
|
||||
/* The single-block sequence-store body only needs this narrow projection of
|
||||
* ZSTD_CCtx. Matchfinding, sequence-store construction, and split-block
|
||||
* control remain in C. */
|
||||
/* The post-split partition loop only needs this projection of ZSTD_CCtx.
|
||||
* Split discovery remains in the surrounding C function; the Rust body owns
|
||||
* partition accounting, repcode simulation, and per-partition emission. */
|
||||
typedef struct {
|
||||
const SeqStore_t* seqStore;
|
||||
U32* dRep;
|
||||
U32* cRep;
|
||||
const U32* partitions;
|
||||
SeqStore_t* nextSeqStore;
|
||||
SeqStore_t* currSeqStore;
|
||||
ZSTD_compressedBlockState_t** prevCBlock;
|
||||
ZSTD_compressedBlockState_t** nextCBlock;
|
||||
void* tmpWorkspace;
|
||||
size_t tmpWkspSize;
|
||||
SeqCollector* seqCollector;
|
||||
size_t blockSizeMax;
|
||||
int strategy;
|
||||
int disableLiteralCompression;
|
||||
int bmi2;
|
||||
int isFirstBlock;
|
||||
} ZSTD_rust_seqStoreSingleBlockState;
|
||||
size_t ZSTD_rust_compressSeqStoreSingleBlock(
|
||||
const ZSTD_rust_seqStoreSingleBlockState* state,
|
||||
} ZSTD_rust_splitBlockState;
|
||||
size_t ZSTD_rust_compressBlockSplit(
|
||||
const ZSTD_rust_splitBlockState* state,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
U32 lastBlock, U32 isPartition);
|
||||
typedef char ZSTD_rust_seqstore_single_block_state_layout[
|
||||
(offsetof(ZSTD_rust_seqStoreSingleBlockState, seqStore) == 0
|
||||
&& offsetof(ZSTD_rust_seqStoreSingleBlockState, dRep) == sizeof(void*)
|
||||
&& offsetof(ZSTD_rust_seqStoreSingleBlockState, cRep) == 2 * sizeof(void*)
|
||||
&& offsetof(ZSTD_rust_seqStoreSingleBlockState, prevCBlock) == 3 * sizeof(void*)
|
||||
&& offsetof(ZSTD_rust_seqStoreSingleBlockState, nextCBlock) == 4 * sizeof(void*)
|
||||
&& offsetof(ZSTD_rust_seqStoreSingleBlockState, tmpWorkspace) == 5 * sizeof(void*)
|
||||
&& offsetof(ZSTD_rust_seqStoreSingleBlockState, tmpWkspSize) == 6 * sizeof(void*)
|
||||
&& offsetof(ZSTD_rust_seqStoreSingleBlockState, seqCollector) == 7 * sizeof(void*)
|
||||
&& offsetof(ZSTD_rust_seqStoreSingleBlockState, strategy) == 8 * sizeof(void*)
|
||||
&& offsetof(ZSTD_rust_seqStoreSingleBlockState, disableLiteralCompression)
|
||||
== 8 * sizeof(void*) + sizeof(int)
|
||||
&& offsetof(ZSTD_rust_seqStoreSingleBlockState, bmi2)
|
||||
== 8 * sizeof(void*) + 2 * sizeof(int)
|
||||
&& offsetof(ZSTD_rust_seqStoreSingleBlockState, isFirstBlock)
|
||||
== 8 * sizeof(void*) + 3 * sizeof(int)
|
||||
&& sizeof(ZSTD_rust_seqStoreSingleBlockState)
|
||||
== 8 * sizeof(void*) + 4 * sizeof(int))
|
||||
const void* src, size_t blockSize,
|
||||
U32 lastBlock, size_t numSplits);
|
||||
typedef char ZSTD_rust_split_block_state_layout[
|
||||
(offsetof(ZSTD_rust_splitBlockState, seqStore) == 0
|
||||
&& offsetof(ZSTD_rust_splitBlockState, partitions) == sizeof(void*)
|
||||
&& offsetof(ZSTD_rust_splitBlockState, nextSeqStore) == 2 * sizeof(void*)
|
||||
&& offsetof(ZSTD_rust_splitBlockState, currSeqStore) == 3 * sizeof(void*)
|
||||
&& offsetof(ZSTD_rust_splitBlockState, prevCBlock) == 4 * sizeof(void*)
|
||||
&& offsetof(ZSTD_rust_splitBlockState, nextCBlock) == 5 * sizeof(void*)
|
||||
&& offsetof(ZSTD_rust_splitBlockState, tmpWorkspace) == 6 * sizeof(void*)
|
||||
&& offsetof(ZSTD_rust_splitBlockState, tmpWkspSize) == 7 * sizeof(void*)
|
||||
&& offsetof(ZSTD_rust_splitBlockState, seqCollector) == 8 * sizeof(void*)
|
||||
&& offsetof(ZSTD_rust_splitBlockState, blockSizeMax) == 9 * sizeof(void*)
|
||||
&& offsetof(ZSTD_rust_splitBlockState, strategy) == 10 * sizeof(void*)
|
||||
&& offsetof(ZSTD_rust_splitBlockState, disableLiteralCompression)
|
||||
== 10 * sizeof(void*) + sizeof(int)
|
||||
&& offsetof(ZSTD_rust_splitBlockState, bmi2)
|
||||
== 10 * sizeof(void*) + 2 * sizeof(int)
|
||||
&& offsetof(ZSTD_rust_splitBlockState, isFirstBlock)
|
||||
== 10 * sizeof(void*) + 3 * sizeof(int)
|
||||
&& sizeof(ZSTD_rust_splitBlockState)
|
||||
== 10 * sizeof(void*) + 4 * sizeof(int))
|
||||
? 1 : -1];
|
||||
size_t ZSTD_rust_nextInputSizeHint(int inBufferMode,
|
||||
size_t blockSizeMax,
|
||||
@@ -417,14 +421,6 @@ size_t ZSTD_rust_buildBlockEntropyStats(
|
||||
int strategy, int disableLiteralCompression,
|
||||
ZSTD_entropyCTablesMetadata_t* entropyMetadata,
|
||||
void* workspace, size_t wkspSize);
|
||||
size_t ZSTD_rust_estimateBlockSize(
|
||||
const BYTE* literals, size_t litSize,
|
||||
const BYTE* ofCodeTable, const BYTE* llCodeTable,
|
||||
const BYTE* mlCodeTable, size_t nbSeq,
|
||||
const ZSTD_entropyCTables_t* entropy,
|
||||
const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
|
||||
void* workspace, size_t wkspSize,
|
||||
int writeLitEntropy, int writeSeqEntropy);
|
||||
size_t ZSTD_rust_copyBlockSequences(
|
||||
SeqCollector* seqCollector, const SeqStore_t* seqStore,
|
||||
const U32 prevRepcodes[ZSTD_REP_NUM]);
|
||||
@@ -448,11 +444,6 @@ int ZSTD_rust_isRLE(const BYTE* src, size_t length);
|
||||
size_t ZSTD_rust_postProcessSequenceProducerResult(
|
||||
ZSTD_Sequence* outSeqs, size_t nbExternalSeqs,
|
||||
size_t outSeqsCapacity, size_t srcSize);
|
||||
size_t ZSTD_rust_countSeqStoreLiteralsBytes(const SeqStore_t* seqStore);
|
||||
size_t ZSTD_rust_countSeqStoreMatchBytes(const SeqStore_t* seqStore);
|
||||
void ZSTD_rust_deriveSeqStoreChunk(SeqStore_t* resultSeqStore,
|
||||
const SeqStore_t* originalSeqStore,
|
||||
size_t startIdx, size_t endIdx);
|
||||
size_t ZSTD_rust_deriveBlockSplits(
|
||||
U32* partitions, U32 nbSeq,
|
||||
const SeqStore_t* originalSeqStore,
|
||||
@@ -2733,105 +2724,6 @@ size_t ZSTD_buildBlockEntropyStats(
|
||||
workspace, wkspSize);
|
||||
}
|
||||
|
||||
/* The block-size estimator is implemented in Rust; C retains the stateful
|
||||
* block-splitting recursion and the entropy-table ownership. */
|
||||
static size_t
|
||||
ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize,
|
||||
const BYTE* ofCodeTable,
|
||||
const BYTE* llCodeTable,
|
||||
const BYTE* mlCodeTable,
|
||||
size_t nbSeq,
|
||||
const ZSTD_entropyCTables_t* entropy,
|
||||
const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
|
||||
void* workspace, size_t wkspSize,
|
||||
int writeLitEntropy, int writeSeqEntropy)
|
||||
{
|
||||
return ZSTD_rust_estimateBlockSize(
|
||||
literals, litSize,
|
||||
ofCodeTable, llCodeTable, mlCodeTable, nbSeq,
|
||||
entropy, entropyMetadata,
|
||||
workspace, wkspSize,
|
||||
writeLitEntropy, writeSeqEntropy);
|
||||
}
|
||||
|
||||
/* Builds entropy statistics and uses them for blocksize estimation.
|
||||
*
|
||||
* @return: estimated compressed size of the seqStore, or a zstd error.
|
||||
*/
|
||||
UNUSED_ATTR static size_t
|
||||
ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(SeqStore_t* seqStore, ZSTD_CCtx* zc)
|
||||
{
|
||||
ZSTD_entropyCTablesMetadata_t* const entropyMetadata = &zc->blockSplitCtx.entropyMetadata;
|
||||
DEBUGLOG(6, "ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()");
|
||||
FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(seqStore,
|
||||
&zc->blockState.prevCBlock->entropy,
|
||||
&zc->blockState.nextCBlock->entropy,
|
||||
&zc->appliedParams,
|
||||
entropyMetadata,
|
||||
zc->tmpWorkspace, zc->tmpWkspSize), "");
|
||||
return ZSTD_estimateBlockSize(
|
||||
seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart),
|
||||
seqStore->ofCode, seqStore->llCode, seqStore->mlCode,
|
||||
(size_t)(seqStore->sequences - seqStore->sequencesStart),
|
||||
&zc->blockState.nextCBlock->entropy,
|
||||
entropyMetadata,
|
||||
zc->tmpWorkspace, zc->tmpWkspSize,
|
||||
(int)(entropyMetadata->hufMetadata.hType == set_compressed), 1);
|
||||
}
|
||||
|
||||
/* Returns literals bytes represented in a seqStore */
|
||||
static size_t ZSTD_countSeqStoreLiteralsBytes(const SeqStore_t* const seqStore)
|
||||
{
|
||||
return ZSTD_rust_countSeqStoreLiteralsBytes(seqStore);
|
||||
}
|
||||
|
||||
/* Returns match bytes represented in a seqStore */
|
||||
static size_t ZSTD_countSeqStoreMatchBytes(const SeqStore_t* const seqStore)
|
||||
{
|
||||
return ZSTD_rust_countSeqStoreMatchBytes(seqStore);
|
||||
}
|
||||
|
||||
/* Derives the seqStore that is a chunk of the originalSeqStore from [startIdx, endIdx).
|
||||
* Stores the result in resultSeqStore.
|
||||
*/
|
||||
static void ZSTD_deriveSeqStoreChunk(SeqStore_t* resultSeqStore,
|
||||
const SeqStore_t* originalSeqStore,
|
||||
size_t startIdx, size_t endIdx)
|
||||
{
|
||||
ZSTD_rust_deriveSeqStoreChunk(resultSeqStore, originalSeqStore,
|
||||
startIdx, endIdx);
|
||||
}
|
||||
|
||||
/* ZSTD_compressSeqStore_singleBlock():
|
||||
* Compresses a seqStore into a block with a block header, into the buffer dst.
|
||||
*
|
||||
* Returns the total size of that block (including header) or a ZSTD error code.
|
||||
*/
|
||||
static size_t
|
||||
ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc,
|
||||
const SeqStore_t* const seqStore,
|
||||
Repcodes_t* const dRep, Repcodes_t* const cRep,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
U32 lastBlock, U32 isPartition)
|
||||
{
|
||||
ZSTD_rust_seqStoreSingleBlockState state;
|
||||
state.seqStore = seqStore;
|
||||
state.dRep = dRep->rep;
|
||||
state.cRep = cRep->rep;
|
||||
state.prevCBlock = &zc->blockState.prevCBlock;
|
||||
state.nextCBlock = &zc->blockState.nextCBlock;
|
||||
state.tmpWorkspace = zc->tmpWorkspace;
|
||||
state.tmpWkspSize = zc->tmpWkspSize;
|
||||
state.seqCollector = &zc->seqCollector;
|
||||
state.strategy = (int)zc->appliedParams.cParams.strategy;
|
||||
state.disableLiteralCompression = ZSTD_literalsCompressionIsDisabled(&zc->appliedParams);
|
||||
state.bmi2 = zc->bmi2;
|
||||
state.isFirstBlock = zc->isFirstBlock;
|
||||
return ZSTD_rust_compressSeqStoreSingleBlock(
|
||||
&state, dst, dstCapacity, src, srcSize, lastBlock, isPartition);
|
||||
}
|
||||
|
||||
/* Base recursive function.
|
||||
* Populates a table with intra-block partition indices that can improve compression ratio.
|
||||
*
|
||||
@@ -2865,91 +2757,29 @@ ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc,
|
||||
const void* src, size_t blockSize,
|
||||
U32 lastBlock, U32 nbSeq)
|
||||
{
|
||||
size_t cSize = 0;
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
BYTE* op = (BYTE*)dst;
|
||||
size_t i = 0;
|
||||
size_t srcBytesTotal = 0;
|
||||
ZSTD_rust_splitBlockState state;
|
||||
U32* const partitions = zc->blockSplitCtx.partitions; /* splits plus the terminal boundary */
|
||||
SeqStore_t* const nextSeqStore = &zc->blockSplitCtx.nextSeqStore;
|
||||
SeqStore_t* const currSeqStore = &zc->blockSplitCtx.currSeqStore;
|
||||
size_t const numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq);
|
||||
|
||||
/* If a block is split and some partitions are emitted as RLE/uncompressed, then repcode history
|
||||
* may become invalid. In order to reconcile potentially invalid repcodes, we keep track of two
|
||||
* separate repcode histories that simulate repcode history on compression and decompression side,
|
||||
* and use the histories to determine whether we must replace a particular repcode with its raw offset.
|
||||
*
|
||||
* 1) cRep gets updated for each partition, regardless of whether the block was emitted as uncompressed
|
||||
* or RLE. This allows us to retrieve the offset value that an invalid repcode references within
|
||||
* a nocompress/RLE block.
|
||||
* 2) dRep gets updated only for compressed partitions, and when a repcode gets replaced, will use
|
||||
* the replacement offset value rather than the original repcode to update the repcode history.
|
||||
* dRep also will be the final repcode history sent to the next block.
|
||||
*
|
||||
* See ZSTD_seqStore_resolveOffCodes() for more details.
|
||||
*/
|
||||
Repcodes_t dRep;
|
||||
Repcodes_t cRep;
|
||||
ZSTD_memcpy(dRep.rep, zc->blockState.prevCBlock->rep, sizeof(Repcodes_t));
|
||||
ZSTD_memcpy(cRep.rep, zc->blockState.prevCBlock->rep, sizeof(Repcodes_t));
|
||||
ZSTD_memset(nextSeqStore, 0, sizeof(SeqStore_t));
|
||||
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)",
|
||||
(unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit,
|
||||
(unsigned)zc->blockState.matchState.nextToUpdate);
|
||||
|
||||
if (numSplits == 0) {
|
||||
size_t cSizeSingleBlock =
|
||||
ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore,
|
||||
&dRep, &cRep,
|
||||
op, dstCapacity,
|
||||
ip, blockSize,
|
||||
lastBlock, 0 /* isPartition */);
|
||||
FORWARD_IF_ERROR(cSizeSingleBlock, "Compressing single block from splitBlock_internal() failed!");
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal: No splits");
|
||||
assert(zc->blockSizeMax <= ZSTD_BLOCKSIZE_MAX);
|
||||
assert(cSizeSingleBlock <= zc->blockSizeMax + ZSTD_blockHeaderSize);
|
||||
return cSizeSingleBlock;
|
||||
}
|
||||
|
||||
ZSTD_deriveSeqStoreChunk(currSeqStore, &zc->seqStore, 0, partitions[0]);
|
||||
for (i = 0; i <= numSplits; ++i) {
|
||||
size_t cSizeChunk;
|
||||
U32 const lastPartition = (i == numSplits);
|
||||
U32 lastBlockEntireSrc = 0;
|
||||
|
||||
size_t srcBytes = ZSTD_countSeqStoreLiteralsBytes(currSeqStore) + ZSTD_countSeqStoreMatchBytes(currSeqStore);
|
||||
srcBytesTotal += srcBytes;
|
||||
if (lastPartition) {
|
||||
/* This is the final partition, need to account for possible last literals */
|
||||
srcBytes += blockSize - srcBytesTotal;
|
||||
lastBlockEntireSrc = lastBlock;
|
||||
} else {
|
||||
ZSTD_deriveSeqStoreChunk(nextSeqStore, &zc->seqStore, partitions[i], partitions[i+1]);
|
||||
}
|
||||
|
||||
cSizeChunk = ZSTD_compressSeqStore_singleBlock(zc, currSeqStore,
|
||||
&dRep, &cRep,
|
||||
op, dstCapacity,
|
||||
ip, srcBytes,
|
||||
lastBlockEntireSrc, 1 /* isPartition */);
|
||||
DEBUGLOG(5, "Estimated size: %zu vs %zu : actual size",
|
||||
ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(currSeqStore, zc), cSizeChunk);
|
||||
FORWARD_IF_ERROR(cSizeChunk, "Compressing chunk failed!");
|
||||
|
||||
ip += srcBytes;
|
||||
op += cSizeChunk;
|
||||
dstCapacity -= cSizeChunk;
|
||||
cSize += cSizeChunk;
|
||||
*currSeqStore = *nextSeqStore;
|
||||
assert(cSizeChunk <= zc->blockSizeMax + ZSTD_blockHeaderSize);
|
||||
}
|
||||
/* cRep and dRep may have diverged during the compression.
|
||||
* If so, we use the dRep repcodes for the next block.
|
||||
*/
|
||||
ZSTD_memcpy(zc->blockState.prevCBlock->rep, dRep.rep, sizeof(Repcodes_t));
|
||||
return cSize;
|
||||
state.seqStore = &zc->seqStore;
|
||||
state.partitions = partitions;
|
||||
state.nextSeqStore = &zc->blockSplitCtx.nextSeqStore;
|
||||
state.currSeqStore = &zc->blockSplitCtx.currSeqStore;
|
||||
state.prevCBlock = &zc->blockState.prevCBlock;
|
||||
state.nextCBlock = &zc->blockState.nextCBlock;
|
||||
state.tmpWorkspace = zc->tmpWorkspace;
|
||||
state.tmpWkspSize = zc->tmpWkspSize;
|
||||
state.seqCollector = &zc->seqCollector;
|
||||
state.blockSizeMax = zc->blockSizeMax;
|
||||
state.strategy = (int)zc->appliedParams.cParams.strategy;
|
||||
state.disableLiteralCompression = ZSTD_literalsCompressionIsDisabled(&zc->appliedParams);
|
||||
state.bmi2 = zc->bmi2;
|
||||
state.isFirstBlock = zc->isFirstBlock;
|
||||
return ZSTD_rust_compressBlockSplit(
|
||||
&state, dst, dstCapacity, src, blockSize, lastBlock, numSplits);
|
||||
}
|
||||
|
||||
static size_t
|
||||
|
||||
@@ -38,6 +38,10 @@ zstd ABI:
|
||||
splitter, and exports collected sequences in the public `ZSTD_Sequence`
|
||||
format. Its C shims extract the sequence store, the entropy-table
|
||||
leaves, and the two `ZSTD_CCtx_params` scalars these paths read.
|
||||
- `zstd_compress_block_split` searches for profitable sequence-store
|
||||
partitions, while `zstd_compress` emits those partitions through the
|
||||
Rust single-block serializer. C retains split discovery's context setup
|
||||
and the outer block-dispatch decision.
|
||||
- `zstd_compress_frame` serializes frame headers, skippable frames, and the
|
||||
last empty block; it takes scalar frame parameters so the C-owned
|
||||
`ZSTD_CCtx_params` layout never crosses the language boundary.
|
||||
|
||||
+399
-7
@@ -29,9 +29,11 @@ use crate::zstd_compress_sequences::SeqDef;
|
||||
use crate::zstd_compress_stats::{
|
||||
SeqCollector, SeqStore_t, ZSTD_Sequence, ZSTD_SequencePosition, ZSTD_compressedBlockState_t,
|
||||
ZSTD_entropyCTables_t, ZSTD_rust_confirmRepcodesAndEntropyTables, ZSTD_rust_copyBlockSequences,
|
||||
ZSTD_rust_determineBlockSize, ZSTD_rust_entropyCompressSeqStore, ZSTD_rust_isRLE,
|
||||
ZSTD_rust_maybeRLE, ZSTD_rust_resetSeqStore, ZSTD_rust_seqStore_resolveOffCodes,
|
||||
ZSTD_rust_transferSequencesNoDelim, ZSTD_rust_transferSequencesWBlockDelim,
|
||||
ZSTD_rust_countSeqStoreLiteralsBytes, ZSTD_rust_countSeqStoreMatchBytes,
|
||||
ZSTD_rust_deriveSeqStoreChunk, ZSTD_rust_determineBlockSize, ZSTD_rust_entropyCompressSeqStore,
|
||||
ZSTD_rust_isRLE, ZSTD_rust_maybeRLE, ZSTD_rust_resetSeqStore,
|
||||
ZSTD_rust_seqStore_resolveOffCodes, ZSTD_rust_transferSequencesNoDelim,
|
||||
ZSTD_rust_transferSequencesWBlockDelim,
|
||||
};
|
||||
use crate::zstd_compress_superblock::ZSTD_rust_compressSuperBlock;
|
||||
use std::ffi::c_void;
|
||||
@@ -166,10 +168,9 @@ const _: () = {
|
||||
|
||||
/// Explicit projection of the state used by `ZSTD_compressSeqStore_singleBlock`.
|
||||
///
|
||||
/// Sequence-store construction and the surrounding split-block bookkeeping
|
||||
/// remain in C. Only the sequence store, simulated repcode histories,
|
||||
/// compressed-block state slots, sequence collector, and scalar compression
|
||||
/// settings cross the ABI.
|
||||
/// Sequence-store construction and split discovery remain in C. Only the
|
||||
/// sequence store, simulated repcode histories, compressed-block state slots,
|
||||
/// sequence collector, and scalar compression settings cross the ABI.
|
||||
#[repr(C)]
|
||||
pub struct ZSTD_rust_seqStoreSingleBlockState {
|
||||
seq_store: *const SeqStore_t,
|
||||
@@ -222,6 +223,58 @@ const _: () = {
|
||||
);
|
||||
};
|
||||
|
||||
/// Explicit projection of the state used by the post-split partition loop.
|
||||
///
|
||||
/// The C caller still owns the private compression context and derives the
|
||||
/// partition boundaries. Rust owns the partition accounting, repcode
|
||||
/// simulation, and repeated single-block dispatch.
|
||||
#[repr(C)]
|
||||
pub struct ZSTD_rust_splitBlockState {
|
||||
seq_store: *const SeqStore_t,
|
||||
partitions: *const u32,
|
||||
next_seq_store: *mut SeqStore_t,
|
||||
curr_seq_store: *mut SeqStore_t,
|
||||
prev_c_block: *mut *mut ZSTD_compressedBlockState_t,
|
||||
next_c_block: *mut *mut ZSTD_compressedBlockState_t,
|
||||
tmp_workspace: *mut c_void,
|
||||
tmp_wksp_size: usize,
|
||||
seq_collector: *mut SeqCollector,
|
||||
block_size_max: usize,
|
||||
strategy: c_int,
|
||||
disable_literal_compression: c_int,
|
||||
bmi2: c_int,
|
||||
is_first_block: c_int,
|
||||
}
|
||||
|
||||
const _: () = {
|
||||
assert!(offset_of!(ZSTD_rust_splitBlockState, seq_store) == 0);
|
||||
assert!(offset_of!(ZSTD_rust_splitBlockState, partitions) == size_of::<usize>());
|
||||
assert!(offset_of!(ZSTD_rust_splitBlockState, next_seq_store) == 2 * size_of::<usize>());
|
||||
assert!(offset_of!(ZSTD_rust_splitBlockState, curr_seq_store) == 3 * size_of::<usize>());
|
||||
assert!(offset_of!(ZSTD_rust_splitBlockState, prev_c_block) == 4 * size_of::<usize>());
|
||||
assert!(offset_of!(ZSTD_rust_splitBlockState, next_c_block) == 5 * size_of::<usize>());
|
||||
assert!(offset_of!(ZSTD_rust_splitBlockState, tmp_workspace) == 6 * size_of::<usize>());
|
||||
assert!(offset_of!(ZSTD_rust_splitBlockState, tmp_wksp_size) == 7 * size_of::<usize>());
|
||||
assert!(offset_of!(ZSTD_rust_splitBlockState, seq_collector) == usize::BITS as usize);
|
||||
assert!(offset_of!(ZSTD_rust_splitBlockState, block_size_max) == 9 * size_of::<usize>());
|
||||
assert!(offset_of!(ZSTD_rust_splitBlockState, strategy) == 10 * size_of::<usize>());
|
||||
assert!(
|
||||
offset_of!(ZSTD_rust_splitBlockState, disable_literal_compression)
|
||||
== 10 * size_of::<usize>() + size_of::<c_int>()
|
||||
);
|
||||
assert!(
|
||||
offset_of!(ZSTD_rust_splitBlockState, bmi2)
|
||||
== 10 * size_of::<usize>() + 2 * size_of::<c_int>()
|
||||
);
|
||||
assert!(
|
||||
offset_of!(ZSTD_rust_splitBlockState, is_first_block)
|
||||
== 10 * size_of::<usize>() + 3 * size_of::<c_int>()
|
||||
);
|
||||
assert!(
|
||||
size_of::<ZSTD_rust_splitBlockState>() == 10 * size_of::<usize>() + 4 * size_of::<c_int>()
|
||||
);
|
||||
};
|
||||
|
||||
/// Explicit projection of the state used by the target-sized block body.
|
||||
///
|
||||
/// Sequence-store construction and the matchfinder remain in C. This state
|
||||
@@ -698,6 +751,213 @@ pub unsafe extern "C" fn ZSTD_rust_compressSeqStoreSingleBlock(
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn split_single_block_state(
|
||||
split_state: &ZSTD_rust_splitBlockState,
|
||||
seq_store: *const SeqStore_t,
|
||||
d_rep: &mut [u32; ZSTD_REP_NUM],
|
||||
c_rep: &mut [u32; ZSTD_REP_NUM],
|
||||
) -> ZSTD_rust_seqStoreSingleBlockState {
|
||||
ZSTD_rust_seqStoreSingleBlockState {
|
||||
seq_store,
|
||||
d_rep: d_rep.as_mut_ptr(),
|
||||
c_rep: c_rep.as_mut_ptr(),
|
||||
prev_c_block: split_state.prev_c_block,
|
||||
next_c_block: split_state.next_c_block,
|
||||
tmp_workspace: split_state.tmp_workspace,
|
||||
tmp_wksp_size: split_state.tmp_wksp_size,
|
||||
seq_collector: split_state.seq_collector,
|
||||
strategy: split_state.strategy,
|
||||
disable_literal_compression: split_state.disable_literal_compression,
|
||||
bmi2: split_state.bmi2,
|
||||
is_first_block: split_state.is_first_block,
|
||||
}
|
||||
}
|
||||
|
||||
/// Rust implementation of the post-split partition loop.
|
||||
///
|
||||
/// C retains `ZSTD_deriveBlockSplits()` and the private `ZSTD_CCtx`; Rust
|
||||
/// receives only the sequence-store views and block-emission state needed to
|
||||
/// reproduce the original loop.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
unsafe fn compress_block_split_body_with(
|
||||
state: &ZSTD_rust_splitBlockState,
|
||||
dst: *mut c_void,
|
||||
dst_capacity: usize,
|
||||
src: *const c_void,
|
||||
block_size: usize,
|
||||
last_block: c_uint,
|
||||
num_splits: usize,
|
||||
seams: SingleBlockSeams,
|
||||
) -> usize {
|
||||
if state.seq_store.is_null()
|
||||
|| state.partitions.is_null()
|
||||
|| state.next_seq_store.is_null()
|
||||
|| state.curr_seq_store.is_null()
|
||||
|| state.prev_c_block.is_null()
|
||||
|| state.next_c_block.is_null()
|
||||
|| state.seq_collector.is_null()
|
||||
{
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
}
|
||||
|
||||
let prev_c_block = unsafe { *state.prev_c_block };
|
||||
let next_c_block = unsafe { *state.next_c_block };
|
||||
if prev_c_block.is_null() || next_c_block.is_null() {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
}
|
||||
|
||||
/* cRep and dRep start from the preceding block and diverge only when a
|
||||
* partition is emitted as raw or RLE. */
|
||||
let mut d_rep = [0u32; ZSTD_REP_NUM];
|
||||
unsafe {
|
||||
ptr::copy_nonoverlapping(
|
||||
(*prev_c_block).rep.as_ptr(),
|
||||
d_rep.as_mut_ptr(),
|
||||
ZSTD_REP_NUM,
|
||||
);
|
||||
}
|
||||
let mut c_rep = d_rep;
|
||||
unsafe {
|
||||
ptr::write_bytes(
|
||||
state.next_seq_store.cast::<u8>(),
|
||||
0,
|
||||
size_of::<SeqStore_t>(),
|
||||
);
|
||||
}
|
||||
|
||||
if num_splits == 0 {
|
||||
let single_state = split_single_block_state(state, state.seq_store, &mut d_rep, &mut c_rep);
|
||||
let c_size = unsafe {
|
||||
compress_seq_store_single_block_body_with(
|
||||
&single_state,
|
||||
dst,
|
||||
dst_capacity,
|
||||
src,
|
||||
block_size,
|
||||
last_block,
|
||||
0,
|
||||
seams,
|
||||
)
|
||||
};
|
||||
debug_assert!(state.block_size_max <= ZSTD_BLOCKSIZE_MAX);
|
||||
debug_assert!(
|
||||
c_size <= state.block_size_max.wrapping_add(ZSTD_BLOCK_HEADER_SIZE)
|
||||
|| ERR_isError(c_size)
|
||||
);
|
||||
return c_size;
|
||||
}
|
||||
|
||||
unsafe {
|
||||
ZSTD_rust_deriveSeqStoreChunk(
|
||||
state.curr_seq_store,
|
||||
state.seq_store,
|
||||
0,
|
||||
*state.partitions as usize,
|
||||
);
|
||||
}
|
||||
|
||||
let mut c_size = 0usize;
|
||||
let mut src_bytes_total = 0usize;
|
||||
let mut ip = src.cast::<u8>();
|
||||
let mut op = dst.cast::<u8>();
|
||||
let mut remaining_capacity = dst_capacity;
|
||||
|
||||
for i in 0..=num_splits {
|
||||
let last_partition = i == num_splits;
|
||||
let mut last_block_entire_src = 0;
|
||||
let partition_seq_store = state.curr_seq_store as *const SeqStore_t;
|
||||
let partition_bytes = unsafe {
|
||||
ZSTD_rust_countSeqStoreLiteralsBytes(partition_seq_store)
|
||||
.wrapping_add(ZSTD_rust_countSeqStoreMatchBytes(partition_seq_store))
|
||||
};
|
||||
src_bytes_total = src_bytes_total.wrapping_add(partition_bytes);
|
||||
let src_bytes = if last_partition {
|
||||
last_block_entire_src = last_block;
|
||||
partition_bytes.wrapping_add(block_size.wrapping_sub(src_bytes_total))
|
||||
} else {
|
||||
unsafe {
|
||||
ZSTD_rust_deriveSeqStoreChunk(
|
||||
state.next_seq_store,
|
||||
state.seq_store,
|
||||
*state.partitions.add(i) as usize,
|
||||
*state.partitions.add(i + 1) as usize,
|
||||
);
|
||||
}
|
||||
partition_bytes
|
||||
};
|
||||
|
||||
let single_state =
|
||||
split_single_block_state(state, partition_seq_store, &mut d_rep, &mut c_rep);
|
||||
let c_size_chunk = unsafe {
|
||||
compress_seq_store_single_block_body_with(
|
||||
&single_state,
|
||||
op.cast(),
|
||||
remaining_capacity,
|
||||
ip.cast(),
|
||||
src_bytes,
|
||||
last_block_entire_src,
|
||||
1,
|
||||
seams,
|
||||
)
|
||||
};
|
||||
if ERR_isError(c_size_chunk) {
|
||||
return c_size_chunk;
|
||||
}
|
||||
|
||||
unsafe {
|
||||
ip = ip.add(src_bytes);
|
||||
op = op.add(c_size_chunk);
|
||||
}
|
||||
remaining_capacity = remaining_capacity.wrapping_sub(c_size_chunk);
|
||||
c_size = c_size.wrapping_add(c_size_chunk);
|
||||
unsafe {
|
||||
ptr::copy_nonoverlapping(state.next_seq_store, state.curr_seq_store, 1);
|
||||
}
|
||||
debug_assert!(c_size_chunk <= state.block_size_max.wrapping_add(ZSTD_BLOCK_HEADER_SIZE));
|
||||
}
|
||||
|
||||
let final_prev_c_block = unsafe { *state.prev_c_block };
|
||||
if final_prev_c_block.is_null() {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
}
|
||||
unsafe {
|
||||
ptr::copy_nonoverlapping(
|
||||
d_rep.as_ptr(),
|
||||
(*final_prev_c_block).rep.as_mut_ptr(),
|
||||
ZSTD_REP_NUM,
|
||||
);
|
||||
}
|
||||
c_size
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_rust_compressBlockSplit(
|
||||
state: *const ZSTD_rust_splitBlockState,
|
||||
dst: *mut c_void,
|
||||
dst_capacity: usize,
|
||||
src: *const c_void,
|
||||
block_size: usize,
|
||||
last_block: c_uint,
|
||||
num_splits: usize,
|
||||
) -> usize {
|
||||
if state.is_null() {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
}
|
||||
unsafe {
|
||||
compress_block_split_body_with(
|
||||
&*state,
|
||||
dst,
|
||||
dst_capacity,
|
||||
src,
|
||||
block_size,
|
||||
last_block,
|
||||
num_splits,
|
||||
SingleBlockSeams::production(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Select the strategy used by the simple compression entry points.
|
||||
///
|
||||
/// This is the Rust equivalent of the strategy portion of
|
||||
@@ -3397,6 +3657,138 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
struct SplitBlockTestStore {
|
||||
seq_store: SeqStore_t,
|
||||
_sequences: Box<[SeqDef; 2]>,
|
||||
_ll_code: Box<[u8; 2]>,
|
||||
_ml_code: Box<[u8; 2]>,
|
||||
_of_code: Box<[u8; 2]>,
|
||||
_literals: Box<[u8; 6]>,
|
||||
}
|
||||
|
||||
fn split_block_test_seq_store() -> SplitBlockTestStore {
|
||||
let mut sequences = Box::new([
|
||||
SeqDef {
|
||||
offBase: 4,
|
||||
litLength: 2,
|
||||
mlBase: 0,
|
||||
},
|
||||
SeqDef {
|
||||
offBase: 5,
|
||||
litLength: 2,
|
||||
mlBase: 0,
|
||||
},
|
||||
]);
|
||||
let mut ll_code = Box::new([1u8; 2]);
|
||||
let mut ml_code = Box::new([1u8; 2]);
|
||||
let mut of_code = Box::new([1u8; 2]);
|
||||
let mut literals = Box::new([0u8; 6]);
|
||||
let sequences_start = sequences.as_mut_ptr();
|
||||
let literals_start = literals.as_mut_ptr();
|
||||
let seq_store = SeqStore_t {
|
||||
sequencesStart: sequences_start,
|
||||
sequences: unsafe { sequences_start.add(sequences.len()) },
|
||||
litStart: literals_start,
|
||||
lit: unsafe { literals_start.add(literals.len()) },
|
||||
llCode: ll_code.as_mut_ptr(),
|
||||
mlCode: ml_code.as_mut_ptr(),
|
||||
ofCode: of_code.as_mut_ptr(),
|
||||
maxNbSeq: sequences.len(),
|
||||
maxNbLit: literals.len(),
|
||||
longLengthType: 0,
|
||||
longLengthPos: 0,
|
||||
};
|
||||
SplitBlockTestStore {
|
||||
seq_store,
|
||||
_sequences: sequences,
|
||||
_ll_code: ll_code,
|
||||
_ml_code: ml_code,
|
||||
_of_code: of_code,
|
||||
_literals: literals,
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn split_block_test_state(
|
||||
seq_store: &SeqStore_t,
|
||||
partitions: &[u32],
|
||||
next_seq_store: &mut SeqStore_t,
|
||||
curr_seq_store: &mut SeqStore_t,
|
||||
prev_block: &mut ZSTD_compressedBlockState_t,
|
||||
next_block: &mut ZSTD_compressedBlockState_t,
|
||||
prev_c_block: &mut *mut ZSTD_compressedBlockState_t,
|
||||
next_c_block: &mut *mut ZSTD_compressedBlockState_t,
|
||||
seq_collector: &mut SeqCollector,
|
||||
) -> ZSTD_rust_splitBlockState {
|
||||
*prev_c_block = prev_block;
|
||||
*next_c_block = next_block;
|
||||
ZSTD_rust_splitBlockState {
|
||||
seq_store,
|
||||
partitions: partitions.as_ptr(),
|
||||
next_seq_store,
|
||||
curr_seq_store,
|
||||
prev_c_block,
|
||||
next_c_block,
|
||||
tmp_workspace: ptr::null_mut(),
|
||||
tmp_wksp_size: 0,
|
||||
seq_collector,
|
||||
block_size_max: ZSTD_BLOCKSIZE_MAX,
|
||||
strategy: 0,
|
||||
disable_literal_compression: 0,
|
||||
bmi2: 0,
|
||||
is_first_block: 1,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_block_body_emits_partitions_and_final_literals() {
|
||||
let fixture = split_block_test_seq_store();
|
||||
let partitions = [1, 2];
|
||||
let mut next_seq_store = unsafe { MaybeUninit::<SeqStore_t>::zeroed().assume_init() };
|
||||
let mut curr_seq_store = unsafe { MaybeUninit::<SeqStore_t>::zeroed().assume_init() };
|
||||
let mut prev_block = zeroed_state();
|
||||
let mut next_block = zeroed_state();
|
||||
let mut prev_c_block = ptr::null_mut();
|
||||
let mut next_c_block = ptr::null_mut();
|
||||
let mut seq_collector = SeqCollector {
|
||||
collectSequences: 0,
|
||||
seqStart: ptr::null_mut(),
|
||||
seqIndex: 0,
|
||||
maxSequences: 0,
|
||||
};
|
||||
let state = split_block_test_state(
|
||||
&fixture.seq_store,
|
||||
&partitions,
|
||||
&mut next_seq_store,
|
||||
&mut curr_seq_store,
|
||||
&mut prev_block,
|
||||
&mut next_block,
|
||||
&mut prev_c_block,
|
||||
&mut next_c_block,
|
||||
&mut seq_collector,
|
||||
);
|
||||
let source: Vec<u8> = (0..12).collect();
|
||||
let mut output = [0xa5u8; 32];
|
||||
|
||||
let result = unsafe {
|
||||
compress_block_split_body_with(
|
||||
&state,
|
||||
output.as_mut_ptr().cast(),
|
||||
output.len(),
|
||||
source.as_ptr().cast(),
|
||||
source.len(),
|
||||
1,
|
||||
1,
|
||||
single_block_test_seams(),
|
||||
)
|
||||
};
|
||||
|
||||
assert_eq!(result, 18);
|
||||
assert_eq!(&output[ZSTD_BLOCK_HEADER_SIZE..8], &source[..5]);
|
||||
assert_eq!(&output[8 + ZSTD_BLOCK_HEADER_SIZE..result], &source[5..]);
|
||||
assert!(std::ptr::eq(prev_c_block, &prev_block));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_block_body_propagates_entropy_errors_without_serializing() {
|
||||
let (seq_store, _sequences, _literals) = target_block_test_seq_store();
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
//! Post-block-split partition search.
|
||||
//!
|
||||
//! The C compressor owns `ZSTD_CCtx`, block-state ownership, entropy-table
|
||||
//! selection, and block emission. This module owns only the recursive search
|
||||
//! over shallow `SeqStore_t` views. The estimator is supplied with projected
|
||||
//! entropy state and parameter scalars rather than the private C context.
|
||||
//! selection, outer block dispatch, and split discovery's context setup. This
|
||||
//! module owns the recursive search over shallow `SeqStore_t` views. The
|
||||
//! estimator is supplied with projected entropy state and parameter scalars
|
||||
//! rather than the private C context; post-split partition emission lives in
|
||||
//! `zstd_compress.rs`.
|
||||
|
||||
use crate::errors::ERR_isError;
|
||||
use crate::zstd_compress_stats::{
|
||||
|
||||
@@ -9,9 +9,10 @@
|
||||
//! `ZSTD_copyBlockSequences()`. The `ZSTD_CCtx` and `ZSTD_CCtx_params`
|
||||
//! layouts stay private to C: the C shims extract the sequence store, the
|
||||
//! entropy-table leaves, and the two parameter scalars these paths read
|
||||
//! (the compression strategy and the literals-compression switch). Block
|
||||
//! dispatch and block-splitting recursion remain in C; the pure block-size
|
||||
//! estimator is owned here.
|
||||
//! (the compression strategy and the literals-compression switch). C retains
|
||||
//! outer block dispatch and split discovery; Rust owns the pure block-size
|
||||
//! estimator and the post-split partition emission loop in
|
||||
//! `zstd_compress.rs`.
|
||||
|
||||
use crate::bits::ZSTD_highbit32;
|
||||
use crate::common::{
|
||||
|
||||
Reference in New Issue
Block a user