feat(compress): route split blocks through Rust
Project frame-chunk split blocks into Rust alongside the existing internal and target paths. Rust now builds the sequence store, asks the C-owned block-split context to derive partitions, refreshes the per-block first-block state, and emits the split blocks through the Rust post-build path. Keep the C partition discovery callback and the existing callback fallback as compatibility seams. Preserve the no-compress path without requiring sequence-store cursors that it does not use. Test Plan: - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo fmt --manifest-path rust/Cargo.toml --all -- --check - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --lib 'zstd_compress::tests::frame_chunk' -- --nocapture - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --lib - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040; make -j1 - ulimit -v 41943040; make -j1 -C tests test-zstream ZSTREAM_TESTTIME=-T2s - ulimit -v 41943040; make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests
This commit is contained in:
@@ -819,12 +819,17 @@ typedef struct ZSTD_rust_compressContinueBlockState_s
|
||||
ZSTD_rust_compressContinueBlockState;
|
||||
typedef struct ZSTD_rust_targetCBlockSizeState_s
|
||||
ZSTD_rust_targetCBlockSizeState;
|
||||
typedef struct ZSTD_rust_splitBlockState_s
|
||||
ZSTD_rust_splitBlockState;
|
||||
typedef size_t (*ZSTD_rust_frameChunkCompress_f)(void* context,
|
||||
void* dst,
|
||||
size_t dstCapacity,
|
||||
const void* src,
|
||||
size_t srcSize,
|
||||
U32 lastBlock);
|
||||
typedef size_t (*ZSTD_rust_frameChunkDeriveSplits_f)(void* context,
|
||||
U32* partitions,
|
||||
U32 nbSeq);
|
||||
typedef struct {
|
||||
void* callbackContext;
|
||||
void* tmpWorkspace;
|
||||
@@ -846,6 +851,8 @@ typedef struct {
|
||||
ZSTD_rust_frameChunkCompress_f compressInternal;
|
||||
const ZSTD_rust_compressContinueBlockState* compressInternalState;
|
||||
const ZSTD_rust_targetCBlockSizeState* compressTargetState;
|
||||
const ZSTD_rust_splitBlockState* compressSplitState;
|
||||
ZSTD_rust_frameChunkDeriveSplits_f deriveBlockSplits;
|
||||
} ZSTD_rust_frameChunkState;
|
||||
size_t ZSTD_rust_compressFrameChunk(
|
||||
const ZSTD_rust_frameChunkState* state,
|
||||
@@ -879,8 +886,12 @@ typedef char ZSTD_rust_frame_chunk_state_layout[
|
||||
== 11 * sizeof(void*) + sizeof(S64) + 6 * sizeof(int)
|
||||
&& offsetof(ZSTD_rust_frameChunkState, compressTargetState)
|
||||
== 12 * sizeof(void*) + sizeof(S64) + 6 * sizeof(int)
|
||||
&& offsetof(ZSTD_rust_frameChunkState, compressSplitState)
|
||||
== 13 * sizeof(void*) + sizeof(S64) + 6 * sizeof(int)
|
||||
&& offsetof(ZSTD_rust_frameChunkState, deriveBlockSplits)
|
||||
== 14 * sizeof(void*) + sizeof(S64) + 6 * sizeof(int)
|
||||
&& sizeof(ZSTD_rust_frameChunkState)
|
||||
== 13 * sizeof(void*) + sizeof(S64) + 6 * sizeof(int))
|
||||
== 15 * sizeof(void*) + sizeof(S64) + 6 * sizeof(int))
|
||||
? 1 : -1];
|
||||
|
||||
/* The high-level continue/block entry points are Rust-owned. This projection
|
||||
@@ -1295,7 +1306,7 @@ typedef char ZSTD_rust_target_cblock_state_layout[
|
||||
/* The post-split partition loop only needs this projection of ZSTD_CCtx.
|
||||
* Split discovery remains in the surrounding C function; the Rust body owns
|
||||
* partition accounting, repcode simulation, and per-partition emission. */
|
||||
typedef struct {
|
||||
struct ZSTD_rust_splitBlockState_s {
|
||||
const SeqStore_t* seqStore;
|
||||
const U32* partitions;
|
||||
SeqStore_t* nextSeqStore;
|
||||
@@ -1310,7 +1321,7 @@ typedef struct {
|
||||
int disableLiteralCompression;
|
||||
int bmi2;
|
||||
int isFirstBlock;
|
||||
} ZSTD_rust_splitBlockState;
|
||||
};
|
||||
size_t ZSTD_rust_compressBlockSplit(
|
||||
const ZSTD_rust_splitBlockState* state,
|
||||
void* dst, size_t dstCapacity,
|
||||
@@ -5432,6 +5443,7 @@ typedef struct {
|
||||
ZSTD_rust_blockInternalState blockInternalState;
|
||||
ZSTD_rust_compressContinueBlockState blockState;
|
||||
ZSTD_rust_targetCBlockSizeState targetBlockState;
|
||||
ZSTD_rust_splitBlockState splitBlockState;
|
||||
} ZSTD_rust_compressContinueContext;
|
||||
|
||||
static int ZSTD_rust_overflowCorrect_need(
|
||||
@@ -5561,6 +5573,13 @@ static size_t ZSTD_rust_frameChunk_compressSplit(
|
||||
(ZSTD_CCtx*)context, dst, dstCapacity, src, srcSize, lastBlock);
|
||||
}
|
||||
|
||||
static size_t ZSTD_rust_frameChunk_deriveBlockSplits(
|
||||
void* context, U32* partitions, U32 nbSeq)
|
||||
{
|
||||
return ZSTD_deriveBlockSplits(
|
||||
(ZSTD_CCtx*)context, partitions, nbSeq);
|
||||
}
|
||||
|
||||
static void ZSTD_compressContinue_prepare(
|
||||
ZSTD_CCtx* cctx, size_t blockSizeMax, int checkBlockSize,
|
||||
ZSTD_rust_compressContinueContext* context)
|
||||
@@ -5611,6 +5630,8 @@ static void ZSTD_compressContinue_prepare(
|
||||
context->frameChunkState.compressSplit = ZSTD_rust_frameChunk_compressSplit;
|
||||
context->frameChunkState.compressInternal = NULL;
|
||||
context->frameChunkState.compressTargetState = &context->targetBlockState;
|
||||
context->frameChunkState.compressSplitState = &context->splitBlockState;
|
||||
context->frameChunkState.deriveBlockSplits = ZSTD_rust_frameChunk_deriveBlockSplits;
|
||||
|
||||
ZSTD_initBuildSeqStoreState(cctx, &context->buildSeqStoreState);
|
||||
context->blockInternalState.seqStore = &cctx->seqStore;
|
||||
@@ -5639,6 +5660,21 @@ static void ZSTD_compressContinue_prepare(
|
||||
context->targetBlockState.windowLog = cctx->appliedParams.cParams.windowLog;
|
||||
context->targetBlockState.targetCBlockSize = cctx->appliedParams.targetCBlockSize;
|
||||
context->targetBlockState.isFirstBlock = cctx->isFirstBlock;
|
||||
context->splitBlockState.seqStore = &cctx->seqStore;
|
||||
context->splitBlockState.partitions = cctx->blockSplitCtx.partitions;
|
||||
context->splitBlockState.nextSeqStore = &cctx->blockSplitCtx.nextSeqStore;
|
||||
context->splitBlockState.currSeqStore = &cctx->blockSplitCtx.currSeqStore;
|
||||
context->splitBlockState.prevCBlock = &cctx->blockState.prevCBlock;
|
||||
context->splitBlockState.nextCBlock = &cctx->blockState.nextCBlock;
|
||||
context->splitBlockState.tmpWorkspace = cctx->tmpWorkspace;
|
||||
context->splitBlockState.tmpWkspSize = cctx->tmpWkspSize;
|
||||
context->splitBlockState.seqCollector = &cctx->seqCollector;
|
||||
context->splitBlockState.blockSizeMax = cctx->blockSizeMax;
|
||||
context->splitBlockState.strategy = (int)cctx->appliedParams.cParams.strategy;
|
||||
context->splitBlockState.disableLiteralCompression =
|
||||
ZSTD_literalsCompressionIsDisabled(&cctx->appliedParams);
|
||||
context->splitBlockState.bmi2 = cctx->bmi2;
|
||||
context->splitBlockState.isFirstBlock = cctx->isFirstBlock;
|
||||
|
||||
context->overflowContext.matchState = ms;
|
||||
context->overflowContext.workspace = &cctx->workspace;
|
||||
|
||||
Reference in New Issue
Block a user