ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move frame epilogue into Rust

Browse Source 
Project the end-frame stage, format flags, window log, and XXH64 state into
Rust so the Rust end-of-frame orchestrator computes the checksum and invokes
the existing Rust epilogue serializer directly. Remove the C epilogue and
checksum adapter while preserving callback order, output accounting, stage
transitions, and error precedence.

Test Plan:
- CARGO_BUILD_JOBS=1 cargo test --lib zstd_compress::tests::compress_end -- --nocapture
- CARGO_BUILD_JOBS=1 cargo clippy --all-targets -- -D warnings
- CARGO_BUILD_JOBS=1 cargo test
- CARGO_BUILD_JOBS=1 make -j1
- CARGO_BUILD_JOBS=1 make -j1 -C tests test-zstream ZSTREAM_TESTTIME=-T2s
- CARGO_BUILD_JOBS=1 make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests

All commands were run serially with a 40 GiB virtual-memory cap.
This commit is contained in:
ddidderr
2026-07-19 23:58:06 +02:00
parent d8e8683353
commit 048979aff6
3 changed files with 119 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+30 -43
View File
@@ -39,10 +39,6 @@ size_t ZSTD_rust_writeFrameHeader(void* dst, size_t dstCapacity,
U32 dictID);
void ZSTD_rust_writeBlockHeader(void* op, size_t cSize, size_t blockSize,
U32 lastBlock);
size_t ZSTD_rust_writeEpilogue(void* dst, size_t dstCapacity, int* stage,
int noDictIDFlag, int checksumFlag,
int contentSizeFlag, int format, U32 windowLog,
U32 checksum);
int ZSTD_rust_updateFrameProgression(unsigned long long* consumedSrcSize,
unsigned long long* producedCSize,
unsigned long long pledgedSrcSizePlusOne,
@@ -935,7 +931,8 @@ typedef char ZSTD_rust_compress_continue_state_layout[
? 1 : -1];
/* Rust owns the end-of-frame orchestration. The callbacks retain the
* private CCtx-dependent continue, epilogue, and trace operations in C. */
* private CCtx-dependent continue and trace operations in C; checksum and
* frame-epilogue fields are projected explicitly. */
typedef size_t (*ZSTD_rust_compressEndContinue_f)(void* context,
void* dst,
size_t dstCapacity,
@@ -943,19 +940,21 @@ typedef size_t (*ZSTD_rust_compressEndContinue_f)(void* context,
size_t srcSize,
U32 frame,
U32 lastFrameChunk);
typedef size_t (*ZSTD_rust_compressEndEpilogue_f)(void* context,
void* dst,
size_t dstCapacity);
typedef void (*ZSTD_rust_compressEndTrace_f)(void* context,
size_t extraCSize);
typedef struct {
void* callbackContext;
ZSTD_rust_compressEndContinue_f compressContinue;
ZSTD_rust_compressEndEpilogue_f writeEpilogue;
ZSTD_rust_compressEndTrace_f trace;
unsigned long long* consumedSrcSize;
U64 pledgedSrcSizePlusOne;
int contentSizeFlag;
int* stage;
int noDictIDFlag;
int checksumFlag;
int format;
U32 windowLog;
const XXH64_state_t* checksumState;
} ZSTD_rust_compressEndState;
size_t ZSTD_rust_compressEnd(const ZSTD_rust_compressEndState* state,
void* dst, size_t dstCapacity,
@@ -964,18 +963,28 @@ typedef char ZSTD_rust_compress_end_state_layout[
(offsetof(ZSTD_rust_compressEndState, callbackContext) == 0
&& offsetof(ZSTD_rust_compressEndState, compressContinue)
== sizeof(void*)
&& offsetof(ZSTD_rust_compressEndState, writeEpilogue)
== 2 * sizeof(void*)
&& offsetof(ZSTD_rust_compressEndState, trace)
== 3 * sizeof(void*)
== 2 * sizeof(void*)
&& offsetof(ZSTD_rust_compressEndState, consumedSrcSize)
== 4 * sizeof(void*)
== 3 * sizeof(void*)
&& offsetof(ZSTD_rust_compressEndState, pledgedSrcSizePlusOne)
== 5 * sizeof(void*)
== 4 * sizeof(void*)
&& offsetof(ZSTD_rust_compressEndState, contentSizeFlag)
== 4 * sizeof(void*) + sizeof(U64)
&& offsetof(ZSTD_rust_compressEndState, stage)
== 5 * sizeof(void*) + sizeof(U64)
&& offsetof(ZSTD_rust_compressEndState, noDictIDFlag)
== 6 * sizeof(void*) + sizeof(U64)
&& offsetof(ZSTD_rust_compressEndState, checksumFlag)
== 6 * sizeof(void*) + sizeof(U64) + sizeof(int)
&& offsetof(ZSTD_rust_compressEndState, format)
== 6 * sizeof(void*) + sizeof(U64) + 2 * sizeof(int)
&& offsetof(ZSTD_rust_compressEndState, windowLog)
== 6 * sizeof(void*) + sizeof(U64) + 3 * sizeof(int)
&& offsetof(ZSTD_rust_compressEndState, checksumState)
== 6 * sizeof(void*) + sizeof(U64) + 3 * sizeof(int) + sizeof(U32)
&& sizeof(ZSTD_rust_compressEndState)
== (sizeof(void*) == 8 ? 56 : 32))
== (sizeof(void*) == 8 ? 80 : 48))
? 1 : -1];
/* Rust owns the single-threaded buffered/stable stream state machine. The
@@ -6161,27 +6170,6 @@ size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel)
}
/*! ZSTD_writeEpilogue() :
* Ends a frame.
* @return : nb of bytes written into dst (or an error code) */
static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
{
U32 const checksum = cctx->appliedParams.fParams.checksumFlag
? (U32)XXH64_digest(&cctx->xxhState)
: 0;
DEBUGLOG(4, "ZSTD_writeEpilogue");
if (cctx->appliedParams.fParams.checksumFlag)
DEBUGLOG(4, "ZSTD_writeEpilogue: write checksum : %08X", (unsigned)checksum);
return ZSTD_rust_writeEpilogue(
dst, dstCapacity, (int*)&cctx->stage,
cctx->appliedParams.fParams.noDictIDFlag,
cctx->appliedParams.fParams.checksumFlag,
cctx->appliedParams.fParams.contentSizeFlag,
(int)cctx->appliedParams.format,
cctx->appliedParams.cParams.windowLog,
checksum);
}
void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize)
{
#if ZSTD_TRACE
@@ -6217,12 +6205,6 @@ static size_t ZSTD_rust_compressEnd_continue(
0 /* block size already selected */);
}
static size_t ZSTD_rust_compressEnd_writeEpilogue(
void* context, void* dst, size_t dstCapacity)
{
return ZSTD_writeEpilogue((ZSTD_CCtx*)context, dst, dstCapacity);
}
static void ZSTD_rust_compressEnd_trace(void* context, size_t extraCSize)
{
ZSTD_CCtx_trace((ZSTD_CCtx*)context, extraCSize);
@@ -6235,11 +6217,16 @@ size_t ZSTD_compressEnd_public(ZSTD_CCtx* cctx,
ZSTD_rust_compressEndState state;
state.callbackContext = cctx;
state.compressContinue = ZSTD_rust_compressEnd_continue;
state.writeEpilogue = ZSTD_rust_compressEnd_writeEpilogue;
state.trace = ZSTD_rust_compressEnd_trace;
state.consumedSrcSize = &cctx->consumedSrcSize;
state.pledgedSrcSizePlusOne = cctx->pledgedSrcSizePlusOne;
state.contentSizeFlag = cctx->appliedParams.fParams.contentSizeFlag;
state.stage = (int*)&cctx->stage;
state.noDictIDFlag = cctx->appliedParams.fParams.noDictIDFlag;
state.checksumFlag = cctx->appliedParams.fParams.checksumFlag;
state.format = (int)cctx->appliedParams.format;
state.windowLog = cctx->appliedParams.cParams.windowLog;
state.checksumState = &cctx->xxhState;
return ZSTD_rust_compressEnd(
&state, dst, dstCapacity, src, srcSize);
}