feat(compress): move ordinary block emission into Rust
Move the post-sequence-store body of ZSTD_compressBlock_internal behind a small C/Rust state projection. C continues to build the sequence store and handle the no-compress and sequence-producer error paths, while Rust now owns sequence collection, entropy emission, the legacy non-first-frame RLE gate, compressed-block confirmation, and offcode repeat cleanup. Remove the C wrappers that became dead after those leaves moved behind the Rust body. The first integration run exposed that leaving the old C finalization label in place confirmed compressed block state twice, undoing Rust's pointer swap and breaking a later sparse-file checksum. The C wrapper now returns directly for the Rust-owned path and retains only its C-owned no-compress cleanup. Test Plan: - cargo test --manifest-path rust/Cargo.toml --lib -- --test-threads=1 - cargo check --manifest-path rust/Cargo.toml --lib - cargo clippy --manifest-path rust/Cargo.toml --lib -- -D warnings - cargo clippy --manifest-path rust/Cargo.toml -- -D warnings - cargo +nightly fmt --manifest-path rust/Cargo.toml -- --check - make -B -C lib -j2 lib - make -B -C tests -j2 test-zstd - make -B -C tests -j2 test-cli-tests - ZSTREAM_TESTTIME=-T2s make -B -C tests -j2 test-zstream - FUZZERTEST=-T5s make -B -C tests -j2 test-fuzzer - make -B -C tests/fuzz -j2 all - make -B -C tests/fuzz -j2 sequence_compression_api - cargo clippy --manifest-path rust/Cargo.toml --tests -- -D warnings (pre-existing manual_repeat_n failure) - cargo clippy --manifest-path rust/Cargo.toml --benches -- -D warnings (pre-existing manual_repeat_n failure)
This commit is contained in:
@@ -83,7 +83,7 @@ int ZSTD_rust_dictTooBig(size_t loadedDictSize);
|
|||||||
* Matchfinder/window state, sequence-store construction, and outer repeat-mode
|
* Matchfinder/window state, sequence-store construction, and outer repeat-mode
|
||||||
* cleanup remain in C. */
|
* cleanup remain in C. */
|
||||||
typedef struct {
|
typedef struct {
|
||||||
SeqStore_t* seqStore;
|
const SeqStore_t* seqStore;
|
||||||
ZSTD_compressedBlockState_t** prevCBlock;
|
ZSTD_compressedBlockState_t** prevCBlock;
|
||||||
ZSTD_compressedBlockState_t** nextCBlock;
|
ZSTD_compressedBlockState_t** nextCBlock;
|
||||||
void* tmpWorkspace;
|
void* tmpWorkspace;
|
||||||
@@ -165,6 +165,42 @@ typedef char ZSTD_rust_split_block_state_layout[
|
|||||||
&& sizeof(ZSTD_rust_splitBlockState)
|
&& sizeof(ZSTD_rust_splitBlockState)
|
||||||
== 10 * sizeof(void*) + 4 * sizeof(int))
|
== 10 * sizeof(void*) + 4 * sizeof(int))
|
||||||
? 1 : -1];
|
? 1 : -1];
|
||||||
|
/* The ordinary sequence-block body only needs this projection of ZSTD_CCtx.
|
||||||
|
* Sequence-store construction and the no-compress fallback remain in C. */
|
||||||
|
typedef struct {
|
||||||
|
const SeqStore_t* seqStore;
|
||||||
|
ZSTD_compressedBlockState_t** prevCBlock;
|
||||||
|
ZSTD_compressedBlockState_t** nextCBlock;
|
||||||
|
void* tmpWorkspace;
|
||||||
|
size_t tmpWkspSize;
|
||||||
|
SeqCollector* seqCollector;
|
||||||
|
int strategy;
|
||||||
|
int disableLiteralCompression;
|
||||||
|
int bmi2;
|
||||||
|
int isFirstBlock;
|
||||||
|
} ZSTD_rust_blockInternalState;
|
||||||
|
size_t ZSTD_rust_compressBlockInternal(
|
||||||
|
const ZSTD_rust_blockInternalState* state,
|
||||||
|
void* dst, size_t dstCapacity,
|
||||||
|
const void* src, size_t srcSize,
|
||||||
|
U32 frame);
|
||||||
|
typedef char ZSTD_rust_block_internal_state_layout[
|
||||||
|
(offsetof(ZSTD_rust_blockInternalState, seqStore) == 0
|
||||||
|
&& offsetof(ZSTD_rust_blockInternalState, prevCBlock) == sizeof(void*)
|
||||||
|
&& offsetof(ZSTD_rust_blockInternalState, nextCBlock) == 2 * sizeof(void*)
|
||||||
|
&& offsetof(ZSTD_rust_blockInternalState, tmpWorkspace) == 3 * sizeof(void*)
|
||||||
|
&& offsetof(ZSTD_rust_blockInternalState, tmpWkspSize) == 4 * sizeof(void*)
|
||||||
|
&& offsetof(ZSTD_rust_blockInternalState, seqCollector) == 5 * sizeof(void*)
|
||||||
|
&& offsetof(ZSTD_rust_blockInternalState, strategy) == 6 * sizeof(void*)
|
||||||
|
&& offsetof(ZSTD_rust_blockInternalState, disableLiteralCompression)
|
||||||
|
== 6 * sizeof(void*) + sizeof(int)
|
||||||
|
&& offsetof(ZSTD_rust_blockInternalState, bmi2)
|
||||||
|
== 6 * sizeof(void*) + 2 * sizeof(int)
|
||||||
|
&& offsetof(ZSTD_rust_blockInternalState, isFirstBlock)
|
||||||
|
== 6 * sizeof(void*) + 3 * sizeof(int)
|
||||||
|
&& sizeof(ZSTD_rust_blockInternalState)
|
||||||
|
== 6 * sizeof(void*) + 4 * sizeof(int))
|
||||||
|
? 1 : -1];
|
||||||
size_t ZSTD_rust_nextInputSizeHint(int inBufferMode,
|
size_t ZSTD_rust_nextInputSizeHint(int inBufferMode,
|
||||||
size_t blockSizeMax,
|
size_t blockSizeMax,
|
||||||
size_t stableInNotConsumed,
|
size_t stableInNotConsumed,
|
||||||
@@ -405,15 +441,6 @@ size_t ZSTD_rust_entropyCompressSeqStore_internal(
|
|||||||
int strategy, int disableLiteralCompression,
|
int strategy, int disableLiteralCompression,
|
||||||
void* entropyWorkspace, size_t entropyWkspSize,
|
void* entropyWorkspace, size_t entropyWkspSize,
|
||||||
int bmi2);
|
int bmi2);
|
||||||
size_t ZSTD_rust_entropyCompressSeqStore(
|
|
||||||
const SeqStore_t* seqStorePtr,
|
|
||||||
const ZSTD_entropyCTables_t* prevEntropy,
|
|
||||||
ZSTD_entropyCTables_t* nextEntropy,
|
|
||||||
int strategy, int disableLiteralCompression,
|
|
||||||
void* dst, size_t dstCapacity,
|
|
||||||
size_t srcSize,
|
|
||||||
void* entropyWorkspace, size_t entropyWkspSize,
|
|
||||||
int bmi2);
|
|
||||||
size_t ZSTD_rust_buildBlockEntropyStats(
|
size_t ZSTD_rust_buildBlockEntropyStats(
|
||||||
const SeqStore_t* seqStorePtr,
|
const SeqStore_t* seqStorePtr,
|
||||||
const ZSTD_entropyCTables_t* prevEntropy,
|
const ZSTD_entropyCTables_t* prevEntropy,
|
||||||
@@ -421,9 +448,6 @@ size_t ZSTD_rust_buildBlockEntropyStats(
|
|||||||
int strategy, int disableLiteralCompression,
|
int strategy, int disableLiteralCompression,
|
||||||
ZSTD_entropyCTablesMetadata_t* entropyMetadata,
|
ZSTD_entropyCTablesMetadata_t* entropyMetadata,
|
||||||
void* workspace, size_t wkspSize);
|
void* workspace, size_t wkspSize);
|
||||||
size_t ZSTD_rust_copyBlockSequences(
|
|
||||||
SeqCollector* seqCollector, const SeqStore_t* seqStore,
|
|
||||||
const U32 prevRepcodes[ZSTD_REP_NUM]);
|
|
||||||
void ZSTD_rust_resetCompressedBlockState(ZSTD_compressedBlockState_t* bs);
|
void ZSTD_rust_resetCompressedBlockState(ZSTD_compressedBlockState_t* bs);
|
||||||
void ZSTD_rust_invalidateRepCodes(U32 rep[ZSTD_REP_NUM]);
|
void ZSTD_rust_invalidateRepCodes(U32 rep[ZSTD_REP_NUM]);
|
||||||
typedef char ZSTD_rust_invalidate_rep_count[(ZSTD_REP_NUM == 3) ? 1 : -1];
|
typedef char ZSTD_rust_invalidate_rep_count[(ZSTD_REP_NUM == 3) ? 1 : -1];
|
||||||
@@ -440,7 +464,6 @@ size_t ZSTD_rust_convertSequencesNoRepcodes(
|
|||||||
SeqDef* dstSeqs, const ZSTD_Sequence* inSeqs, size_t nbSequences);
|
SeqDef* dstSeqs, const ZSTD_Sequence* inSeqs, size_t nbSequences);
|
||||||
BlockSummary ZSTD_rust_get1BlockSummary(const ZSTD_Sequence* seqs,
|
BlockSummary ZSTD_rust_get1BlockSummary(const ZSTD_Sequence* seqs,
|
||||||
size_t nbSeqs);
|
size_t nbSeqs);
|
||||||
int ZSTD_rust_isRLE(const BYTE* src, size_t length);
|
|
||||||
size_t ZSTD_rust_postProcessSequenceProducerResult(
|
size_t ZSTD_rust_postProcessSequenceProducerResult(
|
||||||
ZSTD_Sequence* outSeqs, size_t nbExternalSeqs,
|
ZSTD_Sequence* outSeqs, size_t nbExternalSeqs,
|
||||||
size_t outSeqsCapacity, size_t srcSize);
|
size_t outSeqsCapacity, size_t srcSize);
|
||||||
@@ -2309,27 +2332,6 @@ ZSTD_entropyCompressSeqStore_internal(
|
|||||||
entropyWorkspace, entropyWkspSize, bmi2);
|
entropyWorkspace, entropyWkspSize, bmi2);
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t
|
|
||||||
ZSTD_entropyCompressSeqStore(
|
|
||||||
const SeqStore_t* seqStorePtr,
|
|
||||||
const ZSTD_entropyCTables_t* prevEntropy,
|
|
||||||
ZSTD_entropyCTables_t* nextEntropy,
|
|
||||||
const ZSTD_CCtx_params* cctxParams,
|
|
||||||
void* dst, size_t dstCapacity,
|
|
||||||
size_t srcSize,
|
|
||||||
void* entropyWorkspace, size_t entropyWkspSize,
|
|
||||||
int bmi2)
|
|
||||||
{
|
|
||||||
return ZSTD_rust_entropyCompressSeqStore(
|
|
||||||
seqStorePtr, prevEntropy, nextEntropy,
|
|
||||||
(int)cctxParams->cParams.strategy,
|
|
||||||
ZSTD_literalsCompressionIsDisabled(cctxParams),
|
|
||||||
dst, dstCapacity,
|
|
||||||
srcSize,
|
|
||||||
entropyWorkspace, entropyWkspSize,
|
|
||||||
bmi2);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ZSTD_selectBlockCompressor() :
|
/* ZSTD_selectBlockCompressor() :
|
||||||
* Not static, but internal use only (used by long distance matcher)
|
* Not static, but internal use only (used by long distance matcher)
|
||||||
* assumption : strat is a valid strategy */
|
* assumption : strat is a valid strategy */
|
||||||
@@ -2642,12 +2644,6 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
|
|||||||
return ZSTDbss_compress;
|
return ZSTDbss_compress;
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t ZSTD_copyBlockSequences(SeqCollector* seqCollector, const SeqStore_t* seqStore, const U32 prevRepcodes[ZSTD_REP_NUM])
|
|
||||||
{
|
|
||||||
/* The implementation lives in rust/src/zstd_compress_stats.rs. */
|
|
||||||
return ZSTD_rust_copyBlockSequences(seqCollector, seqStore, prevRepcodes);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ZSTD_sequenceBound() lives in rust/src/zstd_compress_api.rs. */
|
/* ZSTD_sequenceBound() lives in rust/src/zstd_compress_api.rs. */
|
||||||
|
|
||||||
size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
|
size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
|
||||||
@@ -2687,11 +2683,6 @@ size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
|
|||||||
|
|
||||||
/* ZSTD_mergeBlockDelimiters() lives in rust/src/zstd_compress_api.rs. */
|
/* ZSTD_mergeBlockDelimiters() lives in rust/src/zstd_compress_api.rs. */
|
||||||
|
|
||||||
/* Unrolled loop to read four size_ts of input at a time. Returns 1 if is RLE, 0 if not. */
|
|
||||||
static int ZSTD_isRLE(const BYTE* src, size_t length) {
|
|
||||||
return ZSTD_rust_isRLE(src, length);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void
|
static void
|
||||||
ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs)
|
ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs)
|
||||||
{
|
{
|
||||||
@@ -2816,14 +2807,7 @@ ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
|
|||||||
void* dst, size_t dstCapacity,
|
void* dst, size_t dstCapacity,
|
||||||
const void* src, size_t srcSize, U32 frame)
|
const void* src, size_t srcSize, U32 frame)
|
||||||
{
|
{
|
||||||
/* This is an estimated upper bound for the length of an rle block.
|
ZSTD_rust_blockInternalState state;
|
||||||
* This isn't the actual upper bound.
|
|
||||||
* Finding the real threshold needs further investigation.
|
|
||||||
*/
|
|
||||||
const U32 rleMaxLength = 25;
|
|
||||||
size_t cSize;
|
|
||||||
const BYTE* ip = (const BYTE*)src;
|
|
||||||
BYTE* op = (BYTE*)dst;
|
|
||||||
DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)",
|
DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)",
|
||||||
(unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit,
|
(unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit,
|
||||||
(unsigned)zc->blockState.matchState.nextToUpdate);
|
(unsigned)zc->blockState.matchState.nextToUpdate);
|
||||||
@@ -2832,51 +2816,24 @@ ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
|
|||||||
FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed");
|
FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed");
|
||||||
if (bss == ZSTDbss_noCompress) {
|
if (bss == ZSTDbss_noCompress) {
|
||||||
RETURN_ERROR_IF(zc->seqCollector.collectSequences, sequenceProducer_failed, "Uncompressible block");
|
RETURN_ERROR_IF(zc->seqCollector.collectSequences, sequenceProducer_failed, "Uncompressible block");
|
||||||
cSize = 0;
|
if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
|
||||||
goto out;
|
zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
|
||||||
|
return 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (zc->seqCollector.collectSequences) {
|
state.seqStore = &zc->seqStore;
|
||||||
FORWARD_IF_ERROR(ZSTD_copyBlockSequences(&zc->seqCollector, ZSTD_getSeqStore(zc), zc->blockState.prevCBlock->rep), "copyBlockSequences failed");
|
state.prevCBlock = &zc->blockState.prevCBlock;
|
||||||
ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
|
state.nextCBlock = &zc->blockState.nextCBlock;
|
||||||
return 0;
|
state.tmpWorkspace = zc->tmpWorkspace;
|
||||||
}
|
state.tmpWkspSize = zc->tmpWkspSize;
|
||||||
|
state.seqCollector = &zc->seqCollector;
|
||||||
/* encode sequences and literals */
|
state.strategy = (int)zc->appliedParams.cParams.strategy;
|
||||||
cSize = ZSTD_entropyCompressSeqStore(&zc->seqStore,
|
state.disableLiteralCompression = ZSTD_literalsCompressionIsDisabled(&zc->appliedParams);
|
||||||
&zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy,
|
state.bmi2 = zc->bmi2;
|
||||||
&zc->appliedParams,
|
state.isFirstBlock = zc->isFirstBlock;
|
||||||
dst, dstCapacity,
|
return ZSTD_rust_compressBlockInternal(
|
||||||
srcSize,
|
&state, dst, dstCapacity, src, srcSize, frame);
|
||||||
zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */,
|
|
||||||
zc->bmi2);
|
|
||||||
|
|
||||||
if (frame &&
|
|
||||||
/* We don't want to emit our first block as a RLE even if it qualifies because
|
|
||||||
* doing so will cause the decoder (cli only) to throw a "should consume all input error."
|
|
||||||
* This is only an issue for zstd <= v1.4.3
|
|
||||||
*/
|
|
||||||
!zc->isFirstBlock &&
|
|
||||||
cSize < rleMaxLength &&
|
|
||||||
ZSTD_isRLE(ip, srcSize))
|
|
||||||
{
|
|
||||||
cSize = 1;
|
|
||||||
op[0] = ip[0];
|
|
||||||
}
|
|
||||||
|
|
||||||
out:
|
|
||||||
if (!ZSTD_isError(cSize) && cSize > 1) {
|
|
||||||
ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
|
|
||||||
}
|
|
||||||
/* We check that dictionaries have offset codes available for the first
|
|
||||||
* block. After the first block, the offcode table might not have large
|
|
||||||
* enough codes to represent the offsets in the data.
|
|
||||||
*/
|
|
||||||
if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
|
|
||||||
zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
|
|
||||||
|
|
||||||
return cSize;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t ZSTD_compressBlock_targetCBlockSize(ZSTD_CCtx* zc,
|
static size_t ZSTD_compressBlock_targetCBlockSize(ZSTD_CCtx* zc,
|
||||||
|
|||||||
+4
-1
@@ -41,7 +41,10 @@ zstd ABI:
|
|||||||
- `zstd_compress_block_split` searches for profitable sequence-store
|
- `zstd_compress_block_split` searches for profitable sequence-store
|
||||||
partitions, while `zstd_compress` emits those partitions through the
|
partitions, while `zstd_compress` emits those partitions through the
|
||||||
Rust single-block serializer. C retains split discovery's context setup
|
Rust single-block serializer. C retains split discovery's context setup
|
||||||
and the outer block-dispatch decision.
|
and the outer block-dispatch decision. `zstd_compress` also owns ordinary
|
||||||
|
sequence-block entropy emission, sequence collection, and the legacy RLE
|
||||||
|
compatibility gate after C builds the sequence store; C retains the
|
||||||
|
sequence-store builder and no-compress fallback.
|
||||||
- `zstd_compress_frame` serializes frame headers, skippable frames, and the
|
- `zstd_compress_frame` serializes frame headers, skippable frames, and the
|
||||||
last empty block; it takes scalar frame parameters so the C-owned
|
last empty block; it takes scalar frame parameters so the C-owned
|
||||||
`ZSTD_CCtx_params` layout never crosses the language boundary.
|
`ZSTD_CCtx_params` layout never crosses the language boundary.
|
||||||
|
|||||||
@@ -275,6 +275,52 @@ const _: () = {
|
|||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/// Explicit projection of the state used by `ZSTD_compressBlock_internal`.
|
||||||
|
///
|
||||||
|
/// Sequence-store construction and the no-compress fallback remain in C.
|
||||||
|
/// Rust owns sequence collection, entropy emission, the legacy first-block
|
||||||
|
/// RLE gate, and the compressed-block state transitions after the store is
|
||||||
|
/// ready.
|
||||||
|
#[repr(C)]
|
||||||
|
pub struct ZSTD_rust_blockInternalState {
|
||||||
|
seq_store: *const SeqStore_t,
|
||||||
|
prev_c_block: *mut *mut ZSTD_compressedBlockState_t,
|
||||||
|
next_c_block: *mut *mut ZSTD_compressedBlockState_t,
|
||||||
|
tmp_workspace: *mut c_void,
|
||||||
|
tmp_wksp_size: usize,
|
||||||
|
seq_collector: *mut SeqCollector,
|
||||||
|
strategy: c_int,
|
||||||
|
disable_literal_compression: c_int,
|
||||||
|
bmi2: c_int,
|
||||||
|
is_first_block: c_int,
|
||||||
|
}
|
||||||
|
|
||||||
|
const _: () = {
|
||||||
|
assert!(offset_of!(ZSTD_rust_blockInternalState, seq_store) == 0);
|
||||||
|
assert!(offset_of!(ZSTD_rust_blockInternalState, prev_c_block) == size_of::<usize>());
|
||||||
|
assert!(offset_of!(ZSTD_rust_blockInternalState, next_c_block) == 2 * size_of::<usize>());
|
||||||
|
assert!(offset_of!(ZSTD_rust_blockInternalState, tmp_workspace) == 3 * size_of::<usize>());
|
||||||
|
assert!(offset_of!(ZSTD_rust_blockInternalState, tmp_wksp_size) == 4 * size_of::<usize>());
|
||||||
|
assert!(offset_of!(ZSTD_rust_blockInternalState, seq_collector) == 5 * size_of::<usize>());
|
||||||
|
assert!(offset_of!(ZSTD_rust_blockInternalState, strategy) == 6 * size_of::<usize>());
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_blockInternalState, disable_literal_compression)
|
||||||
|
== 6 * size_of::<usize>() + size_of::<c_int>()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_blockInternalState, bmi2)
|
||||||
|
== 6 * size_of::<usize>() + 2 * size_of::<c_int>()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_blockInternalState, is_first_block)
|
||||||
|
== 6 * size_of::<usize>() + 3 * size_of::<c_int>()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
size_of::<ZSTD_rust_blockInternalState>()
|
||||||
|
== 6 * size_of::<usize>() + 4 * size_of::<c_int>()
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
/// Explicit projection of the state used by the target-sized block body.
|
/// Explicit projection of the state used by the target-sized block body.
|
||||||
///
|
///
|
||||||
/// Sequence-store construction and the matchfinder remain in C. This state
|
/// Sequence-store construction and the matchfinder remain in C. This state
|
||||||
@@ -398,6 +444,120 @@ impl SingleBlockSeams {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Rust implementation of `ZSTD_compressBlock_internal` after the C caller
|
||||||
|
/// has built the sequence store.
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
unsafe fn compress_block_internal_body_with(
|
||||||
|
state: &ZSTD_rust_blockInternalState,
|
||||||
|
dst: *mut c_void,
|
||||||
|
dst_capacity: usize,
|
||||||
|
src: *const c_void,
|
||||||
|
src_size: usize,
|
||||||
|
frame: c_uint,
|
||||||
|
seams: SingleBlockSeams,
|
||||||
|
) -> usize {
|
||||||
|
const RLE_MAX_LENGTH: usize = 25;
|
||||||
|
const FSE_REPEAT_CHECK: c_int = 1;
|
||||||
|
const FSE_REPEAT_VALID: c_int = 2;
|
||||||
|
|
||||||
|
if state.seq_store.is_null()
|
||||||
|
|| state.prev_c_block.is_null()
|
||||||
|
|| state.next_c_block.is_null()
|
||||||
|
|| state.seq_collector.is_null()
|
||||||
|
{
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
}
|
||||||
|
|
||||||
|
let prev_c_block = unsafe { *state.prev_c_block };
|
||||||
|
let next_c_block = unsafe { *state.next_c_block };
|
||||||
|
if prev_c_block.is_null() || next_c_block.is_null() {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
}
|
||||||
|
|
||||||
|
if unsafe { (*state.seq_collector).collectSequences } != 0 {
|
||||||
|
let result = unsafe {
|
||||||
|
(seams.copy_sequences)(
|
||||||
|
state.seq_collector,
|
||||||
|
state.seq_store,
|
||||||
|
(*prev_c_block).rep.as_ptr(),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
if ERR_isError(result) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
unsafe { (seams.confirm)(state.prev_c_block, state.next_c_block) };
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut c_size = unsafe {
|
||||||
|
(seams.entropy_compress)(
|
||||||
|
state.seq_store,
|
||||||
|
ptr::addr_of!((*prev_c_block).entropy),
|
||||||
|
ptr::addr_of_mut!((*next_c_block).entropy),
|
||||||
|
state.strategy,
|
||||||
|
state.disable_literal_compression,
|
||||||
|
dst,
|
||||||
|
dst_capacity,
|
||||||
|
src_size,
|
||||||
|
state.tmp_workspace,
|
||||||
|
state.tmp_wksp_size,
|
||||||
|
state.bmi2,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
|
||||||
|
if frame != 0
|
||||||
|
&& state.is_first_block == 0
|
||||||
|
&& c_size < RLE_MAX_LENGTH
|
||||||
|
&& unsafe { (seams.is_rle)(src.cast(), src_size) } != 0
|
||||||
|
{
|
||||||
|
/* Preserve the legacy decoder compatibility rule for the first frame
|
||||||
|
* block: later repeated blocks may use the one-byte RLE form. */
|
||||||
|
c_size = 1;
|
||||||
|
unsafe {
|
||||||
|
*dst.cast::<u8>() = *src.cast::<u8>();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !ERR_isError(c_size) && c_size > 1 {
|
||||||
|
unsafe { (seams.confirm)(state.prev_c_block, state.next_c_block) };
|
||||||
|
}
|
||||||
|
|
||||||
|
let prev_c_block = unsafe { *state.prev_c_block };
|
||||||
|
if prev_c_block.is_null() {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
}
|
||||||
|
if unsafe { (*prev_c_block).entropy.fse.offcode_repeatMode } == FSE_REPEAT_VALID {
|
||||||
|
unsafe { (*prev_c_block).entropy.fse.offcode_repeatMode = FSE_REPEAT_CHECK };
|
||||||
|
}
|
||||||
|
|
||||||
|
c_size
|
||||||
|
}
|
||||||
|
|
||||||
|
#[no_mangle]
|
||||||
|
pub unsafe extern "C" fn ZSTD_rust_compressBlockInternal(
|
||||||
|
state: *const ZSTD_rust_blockInternalState,
|
||||||
|
dst: *mut c_void,
|
||||||
|
dst_capacity: usize,
|
||||||
|
src: *const c_void,
|
||||||
|
src_size: usize,
|
||||||
|
frame: c_uint,
|
||||||
|
) -> usize {
|
||||||
|
if state.is_null() {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
}
|
||||||
|
unsafe {
|
||||||
|
compress_block_internal_body_with(
|
||||||
|
&*state,
|
||||||
|
dst,
|
||||||
|
dst_capacity,
|
||||||
|
src,
|
||||||
|
src_size,
|
||||||
|
frame,
|
||||||
|
SingleBlockSeams::production(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
fn sequence_block_action(
|
fn sequence_block_action(
|
||||||
is_first_block: c_int,
|
is_first_block: c_int,
|
||||||
@@ -3657,6 +3817,32 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn block_internal_test_state(
|
||||||
|
seq_store: &mut SeqStore_t,
|
||||||
|
prev_block: &mut ZSTD_compressedBlockState_t,
|
||||||
|
next_block: &mut ZSTD_compressedBlockState_t,
|
||||||
|
prev_c_block: &mut *mut ZSTD_compressedBlockState_t,
|
||||||
|
next_c_block: &mut *mut ZSTD_compressedBlockState_t,
|
||||||
|
seq_collector: &mut SeqCollector,
|
||||||
|
strategy: c_int,
|
||||||
|
is_first_block: c_int,
|
||||||
|
) -> ZSTD_rust_blockInternalState {
|
||||||
|
*prev_c_block = prev_block;
|
||||||
|
*next_c_block = next_block;
|
||||||
|
ZSTD_rust_blockInternalState {
|
||||||
|
seq_store,
|
||||||
|
prev_c_block,
|
||||||
|
next_c_block,
|
||||||
|
tmp_workspace: ptr::null_mut(),
|
||||||
|
tmp_wksp_size: 0,
|
||||||
|
seq_collector,
|
||||||
|
strategy,
|
||||||
|
disable_literal_compression: 0,
|
||||||
|
bmi2: 0,
|
||||||
|
is_first_block,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
struct SplitBlockTestStore {
|
struct SplitBlockTestStore {
|
||||||
seq_store: SeqStore_t,
|
seq_store: SeqStore_t,
|
||||||
_sequences: Box<[SeqDef; 2]>,
|
_sequences: Box<[SeqDef; 2]>,
|
||||||
@@ -3789,6 +3975,147 @@ mod tests {
|
|||||||
assert!(std::ptr::eq(prev_c_block, &prev_block));
|
assert!(std::ptr::eq(prev_c_block, &prev_block));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn block_internal_body_collects_sequences_before_entropy_emission() {
|
||||||
|
let (mut seq_store, _sequences, _literals) = target_block_test_seq_store();
|
||||||
|
let mut prev_block = zeroed_state();
|
||||||
|
prev_block.entropy.fse.offcode_repeatMode = 2;
|
||||||
|
let mut next_block = zeroed_state();
|
||||||
|
let mut prev_c_block = ptr::null_mut();
|
||||||
|
let mut next_c_block = ptr::null_mut();
|
||||||
|
let mut output_sequences = [ZSTD_Sequence {
|
||||||
|
offset: 0xa5,
|
||||||
|
litLength: 0xa5,
|
||||||
|
matchLength: 0xa5,
|
||||||
|
rep: 0xa5,
|
||||||
|
}];
|
||||||
|
let mut seq_collector = SeqCollector {
|
||||||
|
collectSequences: 1,
|
||||||
|
seqStart: output_sequences.as_mut_ptr(),
|
||||||
|
seqIndex: 0,
|
||||||
|
maxSequences: output_sequences.len(),
|
||||||
|
};
|
||||||
|
let state = block_internal_test_state(
|
||||||
|
&mut seq_store,
|
||||||
|
&mut prev_block,
|
||||||
|
&mut next_block,
|
||||||
|
&mut prev_c_block,
|
||||||
|
&mut next_c_block,
|
||||||
|
&mut seq_collector,
|
||||||
|
1,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
let source = [0x5au8; 16];
|
||||||
|
let mut output = [0xa5u8; 16];
|
||||||
|
|
||||||
|
let result = unsafe {
|
||||||
|
compress_block_internal_body_with(
|
||||||
|
&state,
|
||||||
|
output.as_mut_ptr().cast(),
|
||||||
|
output.len(),
|
||||||
|
source.as_ptr().cast(),
|
||||||
|
source.len(),
|
||||||
|
1,
|
||||||
|
single_block_test_seams(),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(result, 0);
|
||||||
|
assert_eq!(seq_collector.seqIndex, 1);
|
||||||
|
assert_eq!(output_sequences[0].litLength, 0);
|
||||||
|
assert!(std::ptr::eq(prev_c_block, &next_block));
|
||||||
|
assert!(std::ptr::eq(next_c_block, &prev_block));
|
||||||
|
assert_eq!(prev_block.entropy.fse.offcode_repeatMode, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn block_internal_body_gates_rle_and_cleans_repeat_mode() {
|
||||||
|
let (mut seq_store, _sequences, _literals) = target_block_test_seq_store();
|
||||||
|
let mut prev_block = zeroed_state();
|
||||||
|
prev_block.entropy.fse.offcode_repeatMode = 2;
|
||||||
|
let mut next_block = zeroed_state();
|
||||||
|
let mut prev_c_block = ptr::null_mut();
|
||||||
|
let mut next_c_block = ptr::null_mut();
|
||||||
|
let mut seq_collector = SeqCollector {
|
||||||
|
collectSequences: 0,
|
||||||
|
seqStart: ptr::null_mut(),
|
||||||
|
seqIndex: 0,
|
||||||
|
maxSequences: 0,
|
||||||
|
};
|
||||||
|
let state = block_internal_test_state(
|
||||||
|
&mut seq_store,
|
||||||
|
&mut prev_block,
|
||||||
|
&mut next_block,
|
||||||
|
&mut prev_c_block,
|
||||||
|
&mut next_c_block,
|
||||||
|
&mut seq_collector,
|
||||||
|
1,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
let source = [0x5au8; 16];
|
||||||
|
let mut output = [0xa5u8; 32];
|
||||||
|
|
||||||
|
let result = unsafe {
|
||||||
|
compress_block_internal_body_with(
|
||||||
|
&state,
|
||||||
|
output.as_mut_ptr().cast(),
|
||||||
|
output.len(),
|
||||||
|
source.as_ptr().cast(),
|
||||||
|
source.len(),
|
||||||
|
1,
|
||||||
|
single_block_test_seams(),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(result, 1);
|
||||||
|
assert_eq!(output[0], source[0]);
|
||||||
|
assert_eq!(prev_block.entropy.fse.offcode_repeatMode, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn block_internal_body_propagates_entropy_error_through_repeat_cleanup() {
|
||||||
|
let (mut seq_store, _sequences, _literals) = target_block_test_seq_store();
|
||||||
|
let mut prev_block = zeroed_state();
|
||||||
|
prev_block.entropy.fse.offcode_repeatMode = 2;
|
||||||
|
let mut next_block = zeroed_state();
|
||||||
|
let mut prev_c_block = ptr::null_mut();
|
||||||
|
let mut next_c_block = ptr::null_mut();
|
||||||
|
let mut seq_collector = SeqCollector {
|
||||||
|
collectSequences: 0,
|
||||||
|
seqStart: ptr::null_mut(),
|
||||||
|
seqIndex: 0,
|
||||||
|
maxSequences: 0,
|
||||||
|
};
|
||||||
|
let state = block_internal_test_state(
|
||||||
|
&mut seq_store,
|
||||||
|
&mut prev_block,
|
||||||
|
&mut next_block,
|
||||||
|
&mut prev_c_block,
|
||||||
|
&mut next_c_block,
|
||||||
|
&mut seq_collector,
|
||||||
|
-1,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
let source = [0x5au8; 16];
|
||||||
|
let mut output = [0xa5u8; 32];
|
||||||
|
|
||||||
|
let result = unsafe {
|
||||||
|
compress_block_internal_body_with(
|
||||||
|
&state,
|
||||||
|
output.as_mut_ptr().cast(),
|
||||||
|
output.len(),
|
||||||
|
source.as_ptr().cast(),
|
||||||
|
source.len(),
|
||||||
|
1,
|
||||||
|
single_block_test_seams(),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(result, ERROR(ZstdErrorCode::Generic));
|
||||||
|
assert_eq!(output, [0xa5u8; 32]);
|
||||||
|
assert_eq!(prev_block.entropy.fse.offcode_repeatMode, 1);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn single_block_body_propagates_entropy_errors_without_serializing() {
|
fn single_block_body_propagates_entropy_errors_without_serializing() {
|
||||||
let (seq_store, _sequences, _literals) = target_block_test_seq_store();
|
let (seq_store, _sequences, _literals) = target_block_test_seq_store();
|
||||||
|
|||||||
Reference in New Issue
Block a user