feat(compress): move sequence block loop into Rust
Move the per-block body of ZSTD_compressSequences_internal behind an explicit Rust projection. The C wrapper still owns context initialization, public API validation, frame headers, checksums, and the private CCtx layout. Rust now owns block sizing and sequence transfer, sequence-store reset, entropy compression, raw/RLE/compressed block selection, block headers, repcode/state swapping, repeat-mode transition, and first-block handling. The bridge passes only the sequence store, block-state pointer slots, workspace, policy scalars, dictionary size, and the isFirstBlock slot. It does not pass a CCtx or C callback across the ABI. The tests cover empty-block headers, capacity errors, first-block RLE restrictions, and entropy fallback decisions. Test Plan: - cargo test --manifest-path rust/Cargo.toml --lib zstd_compress -- --test-threads=1 - cargo clippy --manifest-path rust/Cargo.toml --lib -- -D warnings - cargo +nightly fmt --manifest-path rust/Cargo.toml -- --check - cargo test --manifest-path rust/Cargo.toml --lib -- --test-threads=1 - make -B -C lib -j2 lib - make -B -C tests -j2 test-cli-tests - ZSTREAM_TESTTIME=-T2s make -B -C tests -j2 test-zstream - FUZZERTEST=-T5s make -C tests -j2 test-fuzzer (covers ZSTD_compressSequences at fuzzer test 190) - git diff --cached --check
This commit is contained in:
+79
-198
@@ -400,12 +400,6 @@ void ZSTD_rust_seqStore_resolveOffCodes(U32 dRep[ZSTD_REP_NUM],
|
|||||||
U32 ZSTD_rust_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM],
|
U32 ZSTD_rust_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM],
|
||||||
U32 offBase, U32 ll0);
|
U32 offBase, U32 ll0);
|
||||||
U32 ZSTD_rust_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0);
|
U32 ZSTD_rust_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0);
|
||||||
size_t ZSTD_rust_blockSizeExplicitDelimiter(const ZSTD_Sequence* inSeqs,
|
|
||||||
size_t inSeqsSize, U32 seqIdx);
|
|
||||||
size_t ZSTD_rust_determineBlockSize(int mode, size_t blockSize, size_t remaining,
|
|
||||||
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
|
|
||||||
U32 seqIdx);
|
|
||||||
int ZSTD_rust_selectSequenceCopier(int mode);
|
|
||||||
size_t ZSTD_rust_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
|
size_t ZSTD_rust_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
|
||||||
const void* dict, size_t dictSize);
|
const void* dict, size_t dictSize);
|
||||||
size_t ZSTD_rust_transferSequencesWBlockDelim(
|
size_t ZSTD_rust_transferSequencesWBlockDelim(
|
||||||
@@ -416,19 +410,58 @@ size_t ZSTD_rust_transferSequencesWBlockDelim(
|
|||||||
U32 nextRepcodes[ZSTD_REP_NUM], U32 dictSize,
|
U32 nextRepcodes[ZSTD_REP_NUM], U32 dictSize,
|
||||||
int validateSequences, U32 minMatch, U32 windowLog,
|
int validateSequences, U32 minMatch, U32 windowLog,
|
||||||
int useSequenceProducer);
|
int useSequenceProducer);
|
||||||
size_t ZSTD_rust_transferSequencesNoDelim(
|
|
||||||
SeqStore_t* seqStore, ZSTD_SequencePosition* seqPos,
|
|
||||||
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
|
|
||||||
const BYTE* src, size_t blockSize,
|
|
||||||
const U32 prevRepcodes[ZSTD_REP_NUM],
|
|
||||||
U32 nextRepcodes[ZSTD_REP_NUM], U32 dictSize,
|
|
||||||
int validateSequences, U32 minMatch, U32 windowLog,
|
|
||||||
int useSequenceProducer);
|
|
||||||
size_t ZSTD_rust_optimalBlockSize(const void* src, size_t srcSize,
|
size_t ZSTD_rust_optimalBlockSize(const void* src, size_t srcSize,
|
||||||
size_t blockSizeMax, int splitLevel,
|
size_t blockSizeMax, int splitLevel,
|
||||||
int strategy, S64 savings,
|
int strategy, S64 savings,
|
||||||
void* workspace, size_t workspaceSize);
|
void* workspace, size_t workspaceSize);
|
||||||
|
|
||||||
|
/* The sequence-compression loop receives only the state it actually reads or
|
||||||
|
* updates. In particular, neither ZSTD_CCtx nor a C function pointer crosses
|
||||||
|
* the Rust ABI. */
|
||||||
|
typedef struct {
|
||||||
|
SeqStore_t* seqStore;
|
||||||
|
ZSTD_compressedBlockState_t** prevCBlock;
|
||||||
|
ZSTD_compressedBlockState_t** nextCBlock;
|
||||||
|
void* tmpWorkspace;
|
||||||
|
size_t tmpWkspSize;
|
||||||
|
size_t blockSizeMax;
|
||||||
|
int bmi2;
|
||||||
|
int blockDelimiters;
|
||||||
|
int strategy;
|
||||||
|
int disableLiteralCompression;
|
||||||
|
int searchForExternalRepcodes;
|
||||||
|
int validateSequences;
|
||||||
|
U32 minMatch;
|
||||||
|
U32 windowLog;
|
||||||
|
U32 dictSize;
|
||||||
|
int useSequenceProducer;
|
||||||
|
int* isFirstBlock;
|
||||||
|
} ZSTD_rust_sequenceCompressionState;
|
||||||
|
|
||||||
|
size_t ZSTD_rust_compressSequencesInternal(
|
||||||
|
const ZSTD_rust_sequenceCompressionState* state,
|
||||||
|
void* dst, size_t dstCapacity,
|
||||||
|
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
|
||||||
|
const void* src, size_t srcSize);
|
||||||
|
|
||||||
|
typedef char ZSTD_rust_sequence_state_layout[
|
||||||
|
(offsetof(ZSTD_rust_sequenceCompressionState, seqStore) == 0
|
||||||
|
&& offsetof(ZSTD_rust_sequenceCompressionState, prevCBlock) == sizeof(void*)
|
||||||
|
&& offsetof(ZSTD_rust_sequenceCompressionState, nextCBlock) == 2 * sizeof(void*)
|
||||||
|
&& offsetof(ZSTD_rust_sequenceCompressionState, tmpWorkspace) == 3 * sizeof(void*)
|
||||||
|
&& offsetof(ZSTD_rust_sequenceCompressionState, tmpWkspSize) == 4 * sizeof(void*)
|
||||||
|
&& offsetof(ZSTD_rust_sequenceCompressionState, blockSizeMax) == 5 * sizeof(void*)
|
||||||
|
&& offsetof(ZSTD_rust_sequenceCompressionState, bmi2) == 6 * sizeof(void*)
|
||||||
|
&& offsetof(ZSTD_rust_sequenceCompressionState, minMatch)
|
||||||
|
== 6 * sizeof(void*) + 6 * sizeof(int)
|
||||||
|
&& offsetof(ZSTD_rust_sequenceCompressionState, useSequenceProducer)
|
||||||
|
== 6 * sizeof(void*) + 3 * sizeof(U32) + 6 * sizeof(int)
|
||||||
|
&& offsetof(ZSTD_rust_sequenceCompressionState, isFirstBlock)
|
||||||
|
== 6 * sizeof(void*) + 3 * sizeof(U32) + 7 * sizeof(int)
|
||||||
|
&& sizeof(ZSTD_rust_sequenceCompressionState)
|
||||||
|
== offsetof(ZSTD_rust_sequenceCompressionState, isFirstBlock) + sizeof(void*))
|
||||||
|
? 1 : -1];
|
||||||
|
|
||||||
typedef char ZSTD_rust_stats_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1];
|
typedef char ZSTD_rust_stats_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1];
|
||||||
typedef char ZSTD_rust_stats_block_summary_layout[
|
typedef char ZSTD_rust_stats_block_summary_layout[
|
||||||
(sizeof(BlockSummary) == 3 * sizeof(size_t)) ? 1 : -1];
|
(sizeof(BlockSummary) == 3 * sizeof(size_t)) ? 1 : -1];
|
||||||
@@ -5110,12 +5143,9 @@ static U32 ZSTD_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32
|
|||||||
return ZSTD_rust_finalizeOffBase(rawOffset, rep, ll0);
|
return ZSTD_rust_finalizeOffBase(rawOffset, rep, ll0);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* This function scans through an array of ZSTD_Sequence,
|
/* The explicit-delimiter adapter is also used by the external sequence
|
||||||
* storing the sequences it reads, until it reaches a block delimiter.
|
* producer path. Keep that C-context-facing call site separate from the
|
||||||
* Note that the block delimiter includes the last literals of the block.
|
* Rust-owned compressSequences block loop. */
|
||||||
* @blockSize must be == sum(sequence_lengths).
|
|
||||||
* @returns @blockSize on success, and a ZSTD_error otherwise.
|
|
||||||
*/
|
|
||||||
static size_t
|
static size_t
|
||||||
ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx,
|
ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx,
|
||||||
ZSTD_SequencePosition* seqPos,
|
ZSTD_SequencePosition* seqPos,
|
||||||
@@ -5143,78 +5173,6 @@ ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx,
|
|||||||
ZSTD_hasExtSeqProd(&cctx->appliedParams));
|
ZSTD_hasExtSeqProd(&cctx->appliedParams));
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
* This function attempts to scan through @blockSize bytes in @src
|
|
||||||
* represented by the sequences in @inSeqs,
|
|
||||||
* storing any (partial) sequences.
|
|
||||||
*
|
|
||||||
* Occasionally, we may want to reduce the actual number of bytes consumed from @src
|
|
||||||
* to avoid splitting a match, notably if it would produce a match smaller than MINMATCH.
|
|
||||||
*
|
|
||||||
* @returns the number of bytes consumed from @src, necessarily <= @blockSize.
|
|
||||||
* Otherwise, it may return a ZSTD error if something went wrong.
|
|
||||||
*/
|
|
||||||
static size_t
|
|
||||||
ZSTD_transferSequences_noDelim(ZSTD_CCtx* cctx,
|
|
||||||
ZSTD_SequencePosition* seqPos,
|
|
||||||
const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
|
|
||||||
const void* src, size_t blockSize,
|
|
||||||
ZSTD_ParamSwitch_e externalRepSearch)
|
|
||||||
{
|
|
||||||
size_t dictSize;
|
|
||||||
|
|
||||||
if (cctx->cdict) {
|
|
||||||
dictSize = cctx->cdict->dictContentSize;
|
|
||||||
} else if (cctx->prefixDict.dict) {
|
|
||||||
dictSize = cctx->prefixDict.dictSize;
|
|
||||||
} else {
|
|
||||||
dictSize = 0;
|
|
||||||
}
|
|
||||||
(void)externalRepSearch;
|
|
||||||
return ZSTD_rust_transferSequencesNoDelim(
|
|
||||||
&cctx->seqStore, seqPos, inSeqs, inSeqsSize,
|
|
||||||
(const BYTE*)src, blockSize,
|
|
||||||
cctx->blockState.prevCBlock->rep,
|
|
||||||
cctx->blockState.nextCBlock->rep, (U32)dictSize,
|
|
||||||
cctx->appliedParams.validateSequences,
|
|
||||||
cctx->appliedParams.cParams.minMatch,
|
|
||||||
cctx->appliedParams.cParams.windowLog,
|
|
||||||
ZSTD_hasExtSeqProd(&cctx->appliedParams));
|
|
||||||
}
|
|
||||||
|
|
||||||
/* @seqPos represents a position within @inSeqs,
|
|
||||||
* it is read and updated by this function,
|
|
||||||
* once the goal to produce a block of size @blockSize is reached.
|
|
||||||
* @return: nb of bytes consumed from @src, necessarily <= @blockSize.
|
|
||||||
*/
|
|
||||||
typedef size_t (*ZSTD_SequenceCopier_f)(ZSTD_CCtx* cctx,
|
|
||||||
ZSTD_SequencePosition* seqPos,
|
|
||||||
const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
|
|
||||||
const void* src, size_t blockSize,
|
|
||||||
ZSTD_ParamSwitch_e externalRepSearch);
|
|
||||||
|
|
||||||
static ZSTD_SequenceCopier_f ZSTD_selectSequenceCopier(ZSTD_SequenceFormat_e mode)
|
|
||||||
{
|
|
||||||
switch (ZSTD_rust_selectSequenceCopier((int)mode)) {
|
|
||||||
case ZSTD_sf_explicitBlockDelimiters:
|
|
||||||
return ZSTD_transferSequences_wBlockDelim;
|
|
||||||
case ZSTD_sf_noBlockDelimiters:
|
|
||||||
default:
|
|
||||||
return ZSTD_transferSequences_noDelim;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static size_t determine_blockSize(ZSTD_SequenceFormat_e mode,
|
|
||||||
size_t blockSize, size_t remaining,
|
|
||||||
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
|
|
||||||
ZSTD_SequencePosition seqPos)
|
|
||||||
{
|
|
||||||
DEBUGLOG(6, "determine_blockSize : remainingSize = %zu", remaining);
|
|
||||||
assert(mode == ZSTD_sf_noBlockDelimiters || mode == ZSTD_sf_explicitBlockDelimiters);
|
|
||||||
return ZSTD_rust_determineBlockSize((int)mode, blockSize, remaining,
|
|
||||||
inSeqs, inSeqsSize, seqPos.idx);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Compress all provided sequences, block-by-block.
|
/* Compress all provided sequences, block-by-block.
|
||||||
*
|
*
|
||||||
* Returns the cumulative size of all compressed blocks (including their headers),
|
* Returns the cumulative size of all compressed blocks (including their headers),
|
||||||
@@ -5226,115 +5184,38 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
|
|||||||
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
|
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
|
||||||
const void* src, size_t srcSize)
|
const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
size_t cSize = 0;
|
U32 dictSize;
|
||||||
size_t remaining = srcSize;
|
ZSTD_rust_sequenceCompressionState state;
|
||||||
ZSTD_SequencePosition seqPos = {0, 0, 0};
|
|
||||||
|
|
||||||
const BYTE* ip = (BYTE const*)src;
|
if (cctx->cdict) {
|
||||||
BYTE* op = (BYTE*)dst;
|
dictSize = (U32)cctx->cdict->dictContentSize;
|
||||||
ZSTD_SequenceCopier_f const sequenceCopier = ZSTD_selectSequenceCopier(cctx->appliedParams.blockDelimiters);
|
} else if (cctx->prefixDict.dict) {
|
||||||
|
dictSize = (U32)cctx->prefixDict.dictSize;
|
||||||
|
} else {
|
||||||
|
dictSize = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
state.seqStore = &cctx->seqStore;
|
||||||
|
state.prevCBlock = &cctx->blockState.prevCBlock;
|
||||||
|
state.nextCBlock = &cctx->blockState.nextCBlock;
|
||||||
|
state.tmpWorkspace = cctx->tmpWorkspace;
|
||||||
|
state.tmpWkspSize = cctx->tmpWkspSize;
|
||||||
|
state.blockSizeMax = cctx->blockSizeMax;
|
||||||
|
state.bmi2 = cctx->bmi2;
|
||||||
|
state.blockDelimiters = (int)cctx->appliedParams.blockDelimiters;
|
||||||
|
state.strategy = (int)cctx->appliedParams.cParams.strategy;
|
||||||
|
state.disableLiteralCompression = ZSTD_literalsCompressionIsDisabled(&cctx->appliedParams);
|
||||||
|
state.searchForExternalRepcodes = (int)cctx->appliedParams.searchForExternalRepcodes;
|
||||||
|
state.validateSequences = cctx->appliedParams.validateSequences;
|
||||||
|
state.minMatch = cctx->appliedParams.cParams.minMatch;
|
||||||
|
state.windowLog = cctx->appliedParams.cParams.windowLog;
|
||||||
|
state.dictSize = dictSize;
|
||||||
|
state.useSequenceProducer = ZSTD_hasExtSeqProd(&cctx->appliedParams);
|
||||||
|
state.isFirstBlock = &cctx->isFirstBlock;
|
||||||
|
|
||||||
DEBUGLOG(4, "ZSTD_compressSequences_internal srcSize: %zu, inSeqsSize: %zu", srcSize, inSeqsSize);
|
DEBUGLOG(4, "ZSTD_compressSequences_internal srcSize: %zu, inSeqsSize: %zu", srcSize, inSeqsSize);
|
||||||
/* Special case: empty frame */
|
return ZSTD_rust_compressSequencesInternal(&state, dst, dstCapacity,
|
||||||
if (remaining == 0) {
|
inSeqs, inSeqsSize, src, srcSize);
|
||||||
U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1);
|
|
||||||
RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "No room for empty frame block header");
|
|
||||||
MEM_writeLE32(op, cBlockHeader24);
|
|
||||||
op += ZSTD_blockHeaderSize;
|
|
||||||
dstCapacity -= ZSTD_blockHeaderSize;
|
|
||||||
cSize += ZSTD_blockHeaderSize;
|
|
||||||
}
|
|
||||||
|
|
||||||
while (remaining) {
|
|
||||||
size_t compressedSeqsSize;
|
|
||||||
size_t cBlockSize;
|
|
||||||
size_t blockSize = determine_blockSize(cctx->appliedParams.blockDelimiters,
|
|
||||||
cctx->blockSizeMax, remaining,
|
|
||||||
inSeqs, inSeqsSize, seqPos);
|
|
||||||
U32 const lastBlock = (blockSize == remaining);
|
|
||||||
FORWARD_IF_ERROR(blockSize, "Error while trying to determine block size");
|
|
||||||
assert(blockSize <= remaining);
|
|
||||||
ZSTD_resetSeqStore(&cctx->seqStore);
|
|
||||||
|
|
||||||
blockSize = sequenceCopier(cctx,
|
|
||||||
&seqPos, inSeqs, inSeqsSize,
|
|
||||||
ip, blockSize,
|
|
||||||
cctx->appliedParams.searchForExternalRepcodes);
|
|
||||||
FORWARD_IF_ERROR(blockSize, "Bad sequence copy");
|
|
||||||
|
|
||||||
/* If blocks are too small, emit as a nocompress block */
|
|
||||||
/* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding
|
|
||||||
* additional 1. We need to revisit and change this logic to be more consistent */
|
|
||||||
if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) {
|
|
||||||
cBlockSize = ZSTD_rust_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
|
|
||||||
FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed");
|
|
||||||
DEBUGLOG(5, "Block too small (%zu): data remains uncompressed: cSize=%zu", blockSize, cBlockSize);
|
|
||||||
cSize += cBlockSize;
|
|
||||||
ip += blockSize;
|
|
||||||
op += cBlockSize;
|
|
||||||
remaining -= blockSize;
|
|
||||||
dstCapacity -= cBlockSize;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "not enough dstCapacity to write a new compressed block");
|
|
||||||
compressedSeqsSize = ZSTD_entropyCompressSeqStore(&cctx->seqStore,
|
|
||||||
&cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy,
|
|
||||||
&cctx->appliedParams,
|
|
||||||
op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize,
|
|
||||||
blockSize,
|
|
||||||
cctx->tmpWorkspace, cctx->tmpWkspSize /* statically allocated in resetCCtx */,
|
|
||||||
cctx->bmi2);
|
|
||||||
FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed");
|
|
||||||
DEBUGLOG(5, "Compressed sequences size: %zu", compressedSeqsSize);
|
|
||||||
|
|
||||||
if (!cctx->isFirstBlock &&
|
|
||||||
ZSTD_maybeRLE(&cctx->seqStore) &&
|
|
||||||
ZSTD_isRLE(ip, blockSize)) {
|
|
||||||
/* Note: don't emit the first block as RLE even if it qualifies because
|
|
||||||
* doing so will cause the decoder (cli <= v1.4.3 only) to throw an (invalid) error
|
|
||||||
* "should consume all input error."
|
|
||||||
*/
|
|
||||||
compressedSeqsSize = 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (compressedSeqsSize == 0) {
|
|
||||||
/* ZSTD_noCompressBlock writes the block header as well */
|
|
||||||
cBlockSize = ZSTD_rust_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
|
|
||||||
FORWARD_IF_ERROR(cBlockSize, "ZSTD_noCompressBlock failed");
|
|
||||||
DEBUGLOG(5, "Writing out nocompress block, size: %zu", cBlockSize);
|
|
||||||
} else if (compressedSeqsSize == 1) {
|
|
||||||
cBlockSize = ZSTD_rust_rleCompressBlock(op, dstCapacity, *ip, blockSize, lastBlock);
|
|
||||||
FORWARD_IF_ERROR(cBlockSize, "ZSTD_rleCompressBlock failed");
|
|
||||||
DEBUGLOG(5, "Writing out RLE block, size: %zu", cBlockSize);
|
|
||||||
} else {
|
|
||||||
/* Error checking and repcodes update */
|
|
||||||
ZSTD_blockState_confirmRepcodesAndEntropyTables(&cctx->blockState);
|
|
||||||
if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
|
|
||||||
cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
|
|
||||||
|
|
||||||
/* Write block header into beginning of block*/
|
|
||||||
ZSTD_rust_writeBlockHeader(op, compressedSeqsSize, blockSize, lastBlock);
|
|
||||||
cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize;
|
|
||||||
DEBUGLOG(5, "Writing out compressed block, size: %zu", cBlockSize);
|
|
||||||
}
|
|
||||||
|
|
||||||
cSize += cBlockSize;
|
|
||||||
|
|
||||||
if (lastBlock) {
|
|
||||||
break;
|
|
||||||
} else {
|
|
||||||
ip += blockSize;
|
|
||||||
op += cBlockSize;
|
|
||||||
remaining -= blockSize;
|
|
||||||
dstCapacity -= cBlockSize;
|
|
||||||
cctx->isFirstBlock = 0;
|
|
||||||
}
|
|
||||||
DEBUGLOG(5, "cSize running total: %zu (remaining dstCapacity=%zu)", cSize, dstCapacity);
|
|
||||||
}
|
|
||||||
|
|
||||||
DEBUGLOG(4, "cSize final total: %zu", cSize);
|
|
||||||
return cSize;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t ZSTD_compressSequences(ZSTD_CCtx* cctx,
|
size_t ZSTD_compressSequences(ZSTD_CCtx* cctx,
|
||||||
|
|||||||
+361
-4
@@ -17,8 +17,8 @@ use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
|
|||||||
#[cfg(not(test))]
|
#[cfg(not(test))]
|
||||||
use crate::zstd_compress_api::ZSTD_compressBound;
|
use crate::zstd_compress_api::ZSTD_compressBound;
|
||||||
use crate::zstd_compress_frame::{
|
use crate::zstd_compress_frame::{
|
||||||
write_raw_block, ZSTD_rust_writeBlockHeader, ZSTD_rust_writeFrameHeader,
|
write_raw_block, ZSTD_rust_noCompressBlock, ZSTD_rust_rleCompressBlock,
|
||||||
ZSTD_writeLastEmptyBlock,
|
ZSTD_rust_writeBlockHeader, ZSTD_rust_writeFrameHeader, ZSTD_writeLastEmptyBlock,
|
||||||
};
|
};
|
||||||
use crate::zstd_compress_literals::min_gain;
|
use crate::zstd_compress_literals::min_gain;
|
||||||
use crate::zstd_compress_params::{
|
use crate::zstd_compress_params::{
|
||||||
@@ -27,10 +27,14 @@ use crate::zstd_compress_params::{
|
|||||||
};
|
};
|
||||||
use crate::zstd_compress_sequences::SeqDef;
|
use crate::zstd_compress_sequences::SeqDef;
|
||||||
use crate::zstd_compress_stats::{
|
use crate::zstd_compress_stats::{
|
||||||
SeqStore_t, ZSTD_compressedBlockState_t, ZSTD_rust_entropyCompressSeqStore, ZSTD_rust_isRLE,
|
SeqStore_t, ZSTD_Sequence, ZSTD_SequencePosition, ZSTD_compressedBlockState_t,
|
||||||
|
ZSTD_rust_confirmRepcodesAndEntropyTables, ZSTD_rust_determineBlockSize,
|
||||||
|
ZSTD_rust_entropyCompressSeqStore, ZSTD_rust_isRLE, ZSTD_rust_maybeRLE,
|
||||||
|
ZSTD_rust_resetSeqStore, ZSTD_rust_transferSequencesNoDelim,
|
||||||
|
ZSTD_rust_transferSequencesWBlockDelim,
|
||||||
};
|
};
|
||||||
use std::ffi::c_void;
|
use std::ffi::c_void;
|
||||||
use std::mem::{size_of, MaybeUninit};
|
use std::mem::{offset_of, size_of, MaybeUninit};
|
||||||
use std::os::raw::{c_int, c_uint};
|
use std::os::raw::{c_int, c_uint};
|
||||||
use std::ptr;
|
use std::ptr;
|
||||||
|
|
||||||
@@ -101,6 +105,64 @@ const ZSTD_CHUNKSIZE_MAX: usize = u32::MAX as usize - ZSTD_CURRENT_MAX;
|
|||||||
#[cfg(not(test))]
|
#[cfg(not(test))]
|
||||||
const ZSTD_E_END: c_int = 2;
|
const ZSTD_E_END: c_int = 2;
|
||||||
|
|
||||||
|
/// Explicit projection of the state used by `ZSTD_compressSequences_internal`.
|
||||||
|
///
|
||||||
|
/// The C context and its function-pointer-bearing parameter structure remain
|
||||||
|
/// private to C. Only the sequence store, compressed-block state slots, and
|
||||||
|
/// scalar policy/workspace fields read by the per-block loop cross the ABI.
|
||||||
|
#[repr(C)]
|
||||||
|
pub struct ZSTD_rust_sequenceCompressionState {
|
||||||
|
seq_store: *mut SeqStore_t,
|
||||||
|
prev_c_block: *mut *mut ZSTD_compressedBlockState_t,
|
||||||
|
next_c_block: *mut *mut ZSTD_compressedBlockState_t,
|
||||||
|
tmp_workspace: *mut c_void,
|
||||||
|
tmp_wksp_size: usize,
|
||||||
|
block_size_max: usize,
|
||||||
|
bmi2: c_int,
|
||||||
|
block_delimiters: c_int,
|
||||||
|
strategy: c_int,
|
||||||
|
disable_literal_compression: c_int,
|
||||||
|
search_for_external_repcodes: c_int,
|
||||||
|
validate_sequences: c_int,
|
||||||
|
min_match: c_uint,
|
||||||
|
window_log: c_uint,
|
||||||
|
dict_size: c_uint,
|
||||||
|
use_sequence_producer: c_int,
|
||||||
|
is_first_block: *mut c_int,
|
||||||
|
}
|
||||||
|
|
||||||
|
const _: () = {
|
||||||
|
assert!(offset_of!(ZSTD_rust_sequenceCompressionState, seq_store) == 0);
|
||||||
|
assert!(offset_of!(ZSTD_rust_sequenceCompressionState, prev_c_block) == size_of::<usize>());
|
||||||
|
assert!(offset_of!(ZSTD_rust_sequenceCompressionState, next_c_block) == 2 * size_of::<usize>());
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_sequenceCompressionState, tmp_workspace) == 3 * size_of::<usize>()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_sequenceCompressionState, tmp_wksp_size) == 4 * size_of::<usize>()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_sequenceCompressionState, block_size_max) == 5 * size_of::<usize>()
|
||||||
|
);
|
||||||
|
assert!(offset_of!(ZSTD_rust_sequenceCompressionState, bmi2) == 6 * size_of::<usize>());
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_sequenceCompressionState, min_match)
|
||||||
|
== 6 * size_of::<usize>() + 6 * size_of::<c_int>()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_sequenceCompressionState, use_sequence_producer)
|
||||||
|
== 6 * size_of::<usize>() + 6 * size_of::<c_int>() + 3 * size_of::<c_uint>()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_sequenceCompressionState, is_first_block)
|
||||||
|
== 6 * size_of::<usize>() + 7 * size_of::<c_int>() + 3 * size_of::<c_uint>()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
size_of::<ZSTD_rust_sequenceCompressionState>()
|
||||||
|
== if size_of::<usize>() == 8 { 96 } else { 68 }
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
#[repr(i32)]
|
#[repr(i32)]
|
||||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||||
enum TargetCBlockAction {
|
enum TargetCBlockAction {
|
||||||
@@ -110,6 +172,30 @@ enum TargetCBlockAction {
|
|||||||
Error = 3,
|
Error = 3,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||||
|
enum SequenceBlockAction {
|
||||||
|
Raw,
|
||||||
|
Rle,
|
||||||
|
Compressed,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn sequence_block_action(
|
||||||
|
is_first_block: c_int,
|
||||||
|
maybe_rle: c_int,
|
||||||
|
is_rle: c_int,
|
||||||
|
compressed_size: usize,
|
||||||
|
) -> SequenceBlockAction {
|
||||||
|
if is_first_block == 0 && maybe_rle != 0 && is_rle != 0 {
|
||||||
|
return SequenceBlockAction::Rle;
|
||||||
|
}
|
||||||
|
match compressed_size {
|
||||||
|
0 => SequenceBlockAction::Raw,
|
||||||
|
1 => SequenceBlockAction::Rle,
|
||||||
|
_ => SequenceBlockAction::Compressed,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
fn target_c_block_size_action(
|
fn target_c_block_size_action(
|
||||||
bss: c_int,
|
bss: c_int,
|
||||||
@@ -952,6 +1038,241 @@ fn ceil_log2(size: usize) -> u32 {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
unsafe fn write_empty_sequence_block(dst: *mut u8, dst_capacity: usize) -> usize {
|
||||||
|
/* Keep the original C helper's four-byte write and capacity check. */
|
||||||
|
if dst_capacity < 4 {
|
||||||
|
return ERROR(ZstdErrorCode::DstSizeTooSmall);
|
||||||
|
}
|
||||||
|
let header = 1u32.to_le_bytes();
|
||||||
|
unsafe { ptr::copy_nonoverlapping(header.as_ptr(), dst, header.len()) };
|
||||||
|
ZSTD_BLOCK_HEADER_SIZE
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rust implementation of the per-block loop from
|
||||||
|
/// `ZSTD_compressSequences_internal()`.
|
||||||
|
///
|
||||||
|
/// Context initialization, frame-header/checksum handling, and public API
|
||||||
|
/// validation remain in C. The projected state keeps the ABI explicit while
|
||||||
|
/// allowing the loop to reuse the existing Rust sequence-transfer, entropy,
|
||||||
|
/// and block-serialization leaves.
|
||||||
|
#[no_mangle]
|
||||||
|
pub unsafe extern "C" fn ZSTD_rust_compressSequencesInternal(
|
||||||
|
state: *const ZSTD_rust_sequenceCompressionState,
|
||||||
|
dst: *mut c_void,
|
||||||
|
dst_capacity: usize,
|
||||||
|
in_seqs: *const ZSTD_Sequence,
|
||||||
|
in_seqs_size: usize,
|
||||||
|
src: *const c_void,
|
||||||
|
src_size: usize,
|
||||||
|
) -> usize {
|
||||||
|
if state.is_null() {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
}
|
||||||
|
let state = unsafe { &*state };
|
||||||
|
if state.seq_store.is_null()
|
||||||
|
|| state.prev_c_block.is_null()
|
||||||
|
|| state.next_c_block.is_null()
|
||||||
|
|| state.is_first_block.is_null()
|
||||||
|
{
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
}
|
||||||
|
if unsafe { (*state.prev_c_block).is_null() || (*state.next_c_block).is_null() } {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut c_size = 0usize;
|
||||||
|
let mut remaining = src_size;
|
||||||
|
let mut seq_pos = ZSTD_SequencePosition::default();
|
||||||
|
let mut ip = src.cast::<u8>();
|
||||||
|
let mut op = dst.cast::<u8>();
|
||||||
|
let mut dst_capacity = dst_capacity;
|
||||||
|
let explicit_delimiters =
|
||||||
|
ZSTD_rust_selectSequenceCopier(state.block_delimiters) == ZSTD_SF_EXPLICIT_BLOCK_DELIMITERS;
|
||||||
|
|
||||||
|
/* Special case: empty frame. */
|
||||||
|
if remaining == 0 {
|
||||||
|
return unsafe { write_empty_sequence_block(op, dst_capacity) };
|
||||||
|
}
|
||||||
|
|
||||||
|
while remaining != 0 {
|
||||||
|
let mut block_size = unsafe {
|
||||||
|
ZSTD_rust_determineBlockSize(
|
||||||
|
state.block_delimiters,
|
||||||
|
state.block_size_max,
|
||||||
|
remaining,
|
||||||
|
in_seqs,
|
||||||
|
in_seqs_size,
|
||||||
|
seq_pos.idx,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let last_block = u32::from(block_size == remaining);
|
||||||
|
if ERR_isError(block_size) {
|
||||||
|
return block_size;
|
||||||
|
}
|
||||||
|
debug_assert!(block_size <= remaining);
|
||||||
|
|
||||||
|
unsafe { ZSTD_rust_resetSeqStore(state.seq_store) };
|
||||||
|
|
||||||
|
let prev_block = unsafe { *state.prev_c_block };
|
||||||
|
let next_block = unsafe { *state.next_c_block };
|
||||||
|
block_size = if explicit_delimiters {
|
||||||
|
unsafe {
|
||||||
|
ZSTD_rust_transferSequencesWBlockDelim(
|
||||||
|
state.seq_store,
|
||||||
|
&mut seq_pos,
|
||||||
|
in_seqs,
|
||||||
|
in_seqs_size,
|
||||||
|
ip,
|
||||||
|
block_size,
|
||||||
|
state.search_for_external_repcodes,
|
||||||
|
(*prev_block).rep.as_ptr(),
|
||||||
|
(*next_block).rep.as_mut_ptr(),
|
||||||
|
state.dict_size,
|
||||||
|
state.validate_sequences,
|
||||||
|
state.min_match,
|
||||||
|
state.window_log,
|
||||||
|
state.use_sequence_producer,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
unsafe {
|
||||||
|
ZSTD_rust_transferSequencesNoDelim(
|
||||||
|
state.seq_store,
|
||||||
|
&mut seq_pos,
|
||||||
|
in_seqs,
|
||||||
|
in_seqs_size,
|
||||||
|
ip,
|
||||||
|
block_size,
|
||||||
|
(*prev_block).rep.as_ptr(),
|
||||||
|
(*next_block).rep.as_mut_ptr(),
|
||||||
|
state.dict_size,
|
||||||
|
state.validate_sequences,
|
||||||
|
state.min_match,
|
||||||
|
state.window_log,
|
||||||
|
state.use_sequence_producer,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if ERR_isError(block_size) {
|
||||||
|
return block_size;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* If blocks are too small, emit as a nocompress block. */
|
||||||
|
if block_size < MIN_COMPRESSIBLE_BLOCK_SIZE {
|
||||||
|
let c_block_size = unsafe {
|
||||||
|
ZSTD_rust_noCompressBlock(
|
||||||
|
op.cast(),
|
||||||
|
dst_capacity,
|
||||||
|
ip.cast(),
|
||||||
|
block_size,
|
||||||
|
last_block,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
if ERR_isError(c_block_size) {
|
||||||
|
return c_block_size;
|
||||||
|
}
|
||||||
|
c_size = c_size.wrapping_add(c_block_size);
|
||||||
|
unsafe {
|
||||||
|
ip = ip.add(block_size);
|
||||||
|
op = op.add(c_block_size);
|
||||||
|
}
|
||||||
|
remaining -= block_size;
|
||||||
|
dst_capacity -= c_block_size;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if dst_capacity < ZSTD_BLOCK_HEADER_SIZE {
|
||||||
|
return ERROR(ZstdErrorCode::DstSizeTooSmall);
|
||||||
|
}
|
||||||
|
|
||||||
|
let compressed_size = unsafe {
|
||||||
|
ZSTD_rust_entropyCompressSeqStore(
|
||||||
|
state.seq_store,
|
||||||
|
ptr::addr_of!((*prev_block).entropy),
|
||||||
|
ptr::addr_of_mut!((*next_block).entropy),
|
||||||
|
state.strategy,
|
||||||
|
state.disable_literal_compression,
|
||||||
|
op.add(ZSTD_BLOCK_HEADER_SIZE).cast(),
|
||||||
|
dst_capacity - ZSTD_BLOCK_HEADER_SIZE,
|
||||||
|
block_size,
|
||||||
|
state.tmp_workspace,
|
||||||
|
state.tmp_wksp_size,
|
||||||
|
state.bmi2,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
if ERR_isError(compressed_size) {
|
||||||
|
return compressed_size;
|
||||||
|
}
|
||||||
|
|
||||||
|
let is_first_block = unsafe { *state.is_first_block };
|
||||||
|
let (maybe_rle, is_rle) = if is_first_block == 0 {
|
||||||
|
let maybe_rle = unsafe { ZSTD_rust_maybeRLE(state.seq_store) };
|
||||||
|
let is_rle = if maybe_rle != 0 {
|
||||||
|
unsafe { ZSTD_rust_isRLE(ip, block_size) }
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
};
|
||||||
|
(maybe_rle, is_rle)
|
||||||
|
} else {
|
||||||
|
(0, 0)
|
||||||
|
};
|
||||||
|
let action = sequence_block_action(is_first_block, maybe_rle, is_rle, compressed_size);
|
||||||
|
|
||||||
|
let c_block_size = match action {
|
||||||
|
SequenceBlockAction::Raw => unsafe {
|
||||||
|
/* ZSTD_noCompressBlock writes the block header as well. */
|
||||||
|
ZSTD_rust_noCompressBlock(
|
||||||
|
op.cast(),
|
||||||
|
dst_capacity,
|
||||||
|
ip.cast(),
|
||||||
|
block_size,
|
||||||
|
last_block,
|
||||||
|
)
|
||||||
|
},
|
||||||
|
SequenceBlockAction::Rle => unsafe {
|
||||||
|
ZSTD_rust_rleCompressBlock(op.cast(), dst_capacity, *ip, block_size, last_block)
|
||||||
|
},
|
||||||
|
SequenceBlockAction::Compressed => {
|
||||||
|
/* Error checking and repcodes update. */
|
||||||
|
unsafe {
|
||||||
|
ZSTD_rust_confirmRepcodesAndEntropyTables(
|
||||||
|
state.prev_c_block,
|
||||||
|
state.next_c_block,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let prev_block = unsafe { *state.prev_c_block };
|
||||||
|
if unsafe { (*prev_block).entropy.fse.offcode_repeatMode } == 2 {
|
||||||
|
unsafe { (*prev_block).entropy.fse.offcode_repeatMode = 1 };
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe {
|
||||||
|
ZSTD_rust_writeBlockHeader(op.cast(), compressed_size, block_size, last_block)
|
||||||
|
};
|
||||||
|
ZSTD_BLOCK_HEADER_SIZE + compressed_size
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if ERR_isError(c_block_size) {
|
||||||
|
return c_block_size;
|
||||||
|
}
|
||||||
|
|
||||||
|
c_size = c_size.wrapping_add(c_block_size);
|
||||||
|
if last_block != 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe {
|
||||||
|
ip = ip.add(block_size);
|
||||||
|
op = op.add(c_block_size);
|
||||||
|
*state.is_first_block = 0;
|
||||||
|
}
|
||||||
|
remaining -= block_size;
|
||||||
|
dst_capacity -= c_block_size;
|
||||||
|
}
|
||||||
|
|
||||||
|
c_size
|
||||||
|
}
|
||||||
|
|
||||||
/// Compress one frame using the already migrated block leaves.
|
/// Compress one frame using the already migrated block leaves.
|
||||||
///
|
///
|
||||||
/// This path owns the match tables and carries their history across the
|
/// This path owns the match tables and carries their history across the
|
||||||
@@ -2242,6 +2563,42 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sequence_block_action_keeps_first_block_from_using_rle() {
|
||||||
|
assert_eq!(
|
||||||
|
sequence_block_action(1, 1, 1, 2),
|
||||||
|
SequenceBlockAction::Compressed
|
||||||
|
);
|
||||||
|
assert_eq!(sequence_block_action(0, 1, 1, 2), SequenceBlockAction::Rle);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sequence_block_action_maps_entropy_fallbacks() {
|
||||||
|
assert_eq!(sequence_block_action(0, 0, 0, 0), SequenceBlockAction::Raw);
|
||||||
|
assert_eq!(sequence_block_action(0, 0, 0, 1), SequenceBlockAction::Rle);
|
||||||
|
assert_eq!(
|
||||||
|
sequence_block_action(0, 0, 0, 2),
|
||||||
|
SequenceBlockAction::Compressed
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn empty_sequence_block_preserves_header_and_capacity_contract() {
|
||||||
|
let mut output = [0xa5; 4];
|
||||||
|
assert_eq!(
|
||||||
|
unsafe { write_empty_sequence_block(output.as_mut_ptr(), output.len()) },
|
||||||
|
ZSTD_BLOCK_HEADER_SIZE
|
||||||
|
);
|
||||||
|
assert_eq!(output, [1, 0, 0, 0]);
|
||||||
|
|
||||||
|
let mut short_output = [0xa5; 3];
|
||||||
|
assert_eq!(
|
||||||
|
unsafe { write_empty_sequence_block(short_output.as_mut_ptr(), short_output.len()) },
|
||||||
|
ERROR(ZstdErrorCode::DstSizeTooSmall)
|
||||||
|
);
|
||||||
|
assert_eq!(short_output, [0xa5; 3]);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn invalidate_rep_codes_clears_all_entries() {
|
fn invalidate_rep_codes_clears_all_entries() {
|
||||||
let mut rep = [11u32, 22, 33];
|
let mut rep = [11u32, 22, 33];
|
||||||
|
|||||||
Reference in New Issue
Block a user