ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(compress): move stable stream init policy to Rust

Browse Source 
ZSTD_compressStream2_c still owns private context mutation, diagnostics, buffer updates, and codec initialization, but its stable-input transparent-initialization decision was a remaining scalar policy island in C. Move the continuation validation, block-size threshold, format-specific progress hint, and wrapping size arithmetic behind a Rust projection. Preserve the original C error messages and return values while keeping all private context state on the C side.\n\nThe bridge uses explicit ABI layout assertions for the mixed pointer/size projection. Focused Rust tests cover buffered and non-continue calls, block-boundary initialization, empty and short stable input, both frame formats, matching continuation state, invalid source/position, and null bridge input.\n\nTest Plan:\n- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo +nightly fmt --manifest-path rust/Cargo.toml --all -- --check\n- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings\n- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings\n- Full capped native and original-test verification follows after the batch is committed.
This commit is contained in:
ddidderr
2026-07-21 06:28:29 +02:00
parent b0e90c39e2
commit 85e9457fcc
2 changed files with 401 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+73 -11
View File
@@ -807,6 +807,51 @@ typedef char ZSTD_rust_compress_stream2_policy_state_layout[
(offsetof(ZSTD_rust_compressStream2PolicyState, endOp) == 0
&& sizeof(ZSTD_rust_compressStream2PolicyState) == sizeof(int))
? 1 : -1];
typedef struct {
int inBufferMode;
int endOp;
size_t inputSize;
size_t stableInNotConsumed;
const void* inputSrc;
size_t inputPos;
const void* expectedInputSrc;
size_t expectedInputSize;
int format;
} ZSTD_rust_compressStream2InitPolicyState;
enum {
ZSTD_RUST_COMPRESS_STREAM2_INIT_POLICY_INITIALIZE = 0,
/* These values are the exact progress hints returned by the C API. */
ZSTD_RUST_COMPRESS_STREAM2_INIT_POLICY_DEFER_MAGICLESS = 2,
ZSTD_RUST_COMPRESS_STREAM2_INIT_POLICY_DEFER_ZSTD = 6,
ZSTD_RUST_COMPRESS_STREAM2_INIT_POLICY_STABLE_SRC_INVALID = -1,
ZSTD_RUST_COMPRESS_STREAM2_INIT_POLICY_STABLE_POS_INVALID = -2
};
int ZSTD_rust_compressStream2InitPolicy(
const ZSTD_rust_compressStream2InitPolicyState* state);
typedef char ZSTD_rust_compress_stream2_init_policy_state_layout[
(offsetof(ZSTD_rust_compressStream2InitPolicyState, inBufferMode) == 0
&& offsetof(ZSTD_rust_compressStream2InitPolicyState, endOp)
== sizeof(int)
&& offsetof(ZSTD_rust_compressStream2InitPolicyState, inputSize)
== 2 * sizeof(int)
&& offsetof(ZSTD_rust_compressStream2InitPolicyState,
stableInNotConsumed)
== 2 * sizeof(int) + sizeof(size_t)
&& offsetof(ZSTD_rust_compressStream2InitPolicyState, inputSrc)
== 2 * sizeof(int) + 2 * sizeof(size_t)
&& offsetof(ZSTD_rust_compressStream2InitPolicyState, inputPos)
== 2 * sizeof(int) + 3 * sizeof(size_t)
&& offsetof(ZSTD_rust_compressStream2InitPolicyState,
expectedInputSrc)
== 2 * sizeof(int) + 4 * sizeof(size_t)
&& offsetof(ZSTD_rust_compressStream2InitPolicyState,
expectedInputSize)
== 2 * sizeof(int) + 5 * sizeof(size_t)
&& offsetof(ZSTD_rust_compressStream2InitPolicyState, format)
== 2 * sizeof(int) + 6 * sizeof(size_t)
&& sizeof(ZSTD_rust_compressStream2InitPolicyState)
== 2 * sizeof(int) + 7 * sizeof(size_t))
? 1 : -1];
int ZSTD_rust_simpleCompress2Level(const void* cctx, size_t srcSize);
typedef int (*ZSTD_rust_simpleCompress2Level_f)(const void* cctx, size_t srcSize);
typedef struct {
@@ -8351,15 +8396,28 @@ size_t ZSTD_compressStream2_c( ZSTD_CCtx* cctx,
if (cctx->streamStage == zcss_init) {
size_t const inputSize = input->size - input->pos; /* no obligation to start from pos==0 */
size_t const totalInputSize = inputSize + cctx->stableIn_notConsumed;
if ( (cctx->requestedParams.inBufferMode == ZSTD_bm_stable) /* input is presumed stable, across invocations */
&& (endOp == ZSTD_e_continue) /* no flush requested, more input to come */
&& (totalInputSize < ZSTD_BLOCKSIZE_MAX) ) { /* not even reached one block yet */
if (cctx->stableIn_notConsumed) { /* not the first time */
/* check stable source guarantees */
RETURN_ERROR_IF(input->src != cctx->expectedInBuffer.src, stabilityCondition_notRespected, "stableInBuffer condition not respected: wrong src pointer");
RETURN_ERROR_IF(input->pos != cctx->expectedInBuffer.size, stabilityCondition_notRespected, "stableInBuffer condition not respected: externally modified pos");
}
ZSTD_rust_compressStream2InitPolicyState const state = {
(int)cctx->requestedParams.inBufferMode,
(int)endOp,
inputSize,
cctx->stableIn_notConsumed,
input->src,
input->pos,
cctx->expectedInBuffer.src,
cctx->expectedInBuffer.size,
(int)cctx->requestedParams.format
};
int const initPolicy = ZSTD_rust_compressStream2InitPolicy(&state);
RETURN_ERROR_IF(
initPolicy == ZSTD_RUST_COMPRESS_STREAM2_INIT_POLICY_STABLE_SRC_INVALID,
stabilityCondition_notRespected,
"stableInBuffer condition not respected: wrong src pointer");
RETURN_ERROR_IF(
initPolicy == ZSTD_RUST_COMPRESS_STREAM2_INIT_POLICY_STABLE_POS_INVALID,
stabilityCondition_notRespected,
"stableInBuffer condition not respected: externally modified pos");
if (initPolicy == ZSTD_RUST_COMPRESS_STREAM2_INIT_POLICY_DEFER_MAGICLESS
|| initPolicy == ZSTD_RUST_COMPRESS_STREAM2_INIT_POLICY_DEFER_ZSTD) {
/* pretend input was consumed, to give a sense forward progress */
input->pos = input->size;
/* save stable inBuffer, for later control, and flush/end */
@@ -8367,9 +8425,13 @@ size_t ZSTD_compressStream2_c( ZSTD_CCtx* cctx,
/* but actually input wasn't consumed, so keep track of position from where compression shall resume */
cctx->stableIn_notConsumed += inputSize;
/* don't initialize yet, wait for the first block of flush() order, for better parameters adaptation */
return ZSTD_FRAMEHEADERSIZE_MIN(cctx->requestedParams.format); /* at least some header to produce */
return (size_t)initPolicy; /* at least some header to produce */
}
FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, totalInputSize), "compressStream2 initialization failed");
FORWARD_IF_ERROR(
ZSTD_CCtx_init_compressStream2(
cctx, endOp,
inputSize + cctx->stableIn_notConsumed),
"compressStream2 initialization failed");
ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */
}
/* end of transparent initialization stage */