ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move CCtx reset storage policy into Rust

Browse Source 
Move the post-match-state CCtx storage reservation and publication policy into
Rust while keeping the private C workspace and context layouts behind narrow
callbacks. The Rust leaf now preserves the sequence, LDM, external-sequence,
literal, buffered-input/output, bucket, and entropy-code allocation order,
including zeroing and reset callbacks. C remains responsible for workspace
resize/layout, private field publication, allocator callbacks, and the LDM
window reset details. ABI layout assertions and fake-callback tests cover
ordinary, LDM/external-producer, and allocation-failure paths.

Test Plan:
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo fmt --manifest-path rust/Cargo.toml
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml cctx_storage --lib (3 passed)
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --lib (683 passed)
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings
- ulimit -v 41943040 && make -B -C programs -j1 zstd (passed; existing fileio const-cast warnings)
- ulimit -v 41943040 && make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s (84 tests and both short fuzz rounds passed; existing zstreamtest warning)
This commit is contained in:
ddidderr
2026-07-19 18:16:44 +02:00
parent 853792f2c4
commit db8c58ac97
3 changed files with 930 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+293 -55
View File
@@ -1274,6 +1274,121 @@ typedef char ZSTD_rust_cctx_reset_plan_layout[
&& offsetof(ZSTD_rustCCtxResetPlan, needsIndexReset) == 8 * sizeof(size_t)
&& sizeof(ZSTD_rustCCtxResetPlan) == 9 * sizeof(size_t))
? 1 : -1];
typedef void (*ZSTD_rust_resetCCtxStorageSetPointer_f)(
void* context, int pointerKind, void* pointer);
typedef void (*ZSTD_rust_resetCCtxStorageSetSize_f)(
void* context, int sizeKind, size_t value);
typedef void (*ZSTD_rust_resetCCtxStorageSetInt_f)(
void* context, int intKind, int value);
typedef void* (*ZSTD_rust_resetCCtxStorageReserve_f)(
void* context, int reserveKind, size_t size);
typedef int (*ZSTD_rust_resetCCtxStorageReserveFailed_f)(void* context);
typedef void (*ZSTD_rust_resetCCtxStorageZero_f)(
void* context, void* pointer, size_t size);
typedef void (*ZSTD_rust_resetCCtxStorageCallback_f)(void* context);
typedef struct {
void* callbackContext;
int ldmEnable;
int hasExtSeqProd;
U32 hashLog;
U32 bucketSizeLog;
size_t blockSize;
size_t maxNbSeq;
size_t maxNbLdmSeq;
size_t maxNbExternalSeq;
size_t buffInSize;
size_t buffOutSize;
size_t seqDefSize;
size_t ldmEntrySize;
size_t rawSeqSize;
size_t externalSequenceSize;
size_t byteSize;
size_t wildcopyOverlength;
int bufferedPolicy;
ZSTD_rust_resetCCtxStorageSetPointer_f setPointer;
ZSTD_rust_resetCCtxStorageSetSize_f setSize;
ZSTD_rust_resetCCtxStorageSetInt_f setInt;
ZSTD_rust_resetCCtxStorageReserve_f reserve;
ZSTD_rust_resetCCtxStorageReserveFailed_f reserveFailed;
ZSTD_rust_resetCCtxStorageZero_f zero;
ZSTD_rust_resetCCtxStorageCallback_f windowInit;
ZSTD_rust_resetCCtxStorageCallback_f resetExternalSequences;
} ZSTD_rust_resetCCtxStorageState;
typedef char ZSTD_rust_reset_cctx_storage_state_layout[
(offsetof(ZSTD_rust_resetCCtxStorageState, callbackContext) == 0
&& offsetof(ZSTD_rust_resetCCtxStorageState, ldmEnable) == sizeof(void*)
&& offsetof(ZSTD_rust_resetCCtxStorageState, hasExtSeqProd)
== sizeof(void*) + sizeof(int)
&& offsetof(ZSTD_rust_resetCCtxStorageState, hashLog)
> offsetof(ZSTD_rust_resetCCtxStorageState, hasExtSeqProd)
&& offsetof(ZSTD_rust_resetCCtxStorageState, bucketSizeLog)
== offsetof(ZSTD_rust_resetCCtxStorageState, hashLog) + sizeof(U32)
&& offsetof(ZSTD_rust_resetCCtxStorageState, blockSize)
> offsetof(ZSTD_rust_resetCCtxStorageState, bucketSizeLog)
&& offsetof(ZSTD_rust_resetCCtxStorageState, maxNbSeq)
== offsetof(ZSTD_rust_resetCCtxStorageState, blockSize) + sizeof(size_t)
&& offsetof(ZSTD_rust_resetCCtxStorageState, maxNbLdmSeq)
== offsetof(ZSTD_rust_resetCCtxStorageState, maxNbSeq) + sizeof(size_t)
&& offsetof(ZSTD_rust_resetCCtxStorageState, maxNbExternalSeq)
== offsetof(ZSTD_rust_resetCCtxStorageState, maxNbLdmSeq) + sizeof(size_t)
&& offsetof(ZSTD_rust_resetCCtxStorageState, buffInSize)
== offsetof(ZSTD_rust_resetCCtxStorageState, maxNbExternalSeq) + sizeof(size_t)
&& offsetof(ZSTD_rust_resetCCtxStorageState, buffOutSize)
== offsetof(ZSTD_rust_resetCCtxStorageState, buffInSize) + sizeof(size_t)
&& offsetof(ZSTD_rust_resetCCtxStorageState, seqDefSize)
== offsetof(ZSTD_rust_resetCCtxStorageState, buffOutSize) + sizeof(size_t)
&& offsetof(ZSTD_rust_resetCCtxStorageState, ldmEntrySize)
== offsetof(ZSTD_rust_resetCCtxStorageState, seqDefSize) + sizeof(size_t)
&& offsetof(ZSTD_rust_resetCCtxStorageState, rawSeqSize)
== offsetof(ZSTD_rust_resetCCtxStorageState, ldmEntrySize) + sizeof(size_t)
&& offsetof(ZSTD_rust_resetCCtxStorageState, externalSequenceSize)
== offsetof(ZSTD_rust_resetCCtxStorageState, rawSeqSize) + sizeof(size_t)
&& offsetof(ZSTD_rust_resetCCtxStorageState, byteSize)
== offsetof(ZSTD_rust_resetCCtxStorageState, externalSequenceSize) + sizeof(size_t)
&& offsetof(ZSTD_rust_resetCCtxStorageState, wildcopyOverlength)
== offsetof(ZSTD_rust_resetCCtxStorageState, byteSize) + sizeof(size_t)
&& offsetof(ZSTD_rust_resetCCtxStorageState, bufferedPolicy)
> offsetof(ZSTD_rust_resetCCtxStorageState, wildcopyOverlength)
&& offsetof(ZSTD_rust_resetCCtxStorageState, setSize)
== offsetof(ZSTD_rust_resetCCtxStorageState, setPointer) + sizeof(void*)
&& offsetof(ZSTD_rust_resetCCtxStorageState, setInt)
== offsetof(ZSTD_rust_resetCCtxStorageState, setSize) + sizeof(void*)
&& offsetof(ZSTD_rust_resetCCtxStorageState, reserve)
== offsetof(ZSTD_rust_resetCCtxStorageState, setInt) + sizeof(void*)
&& sizeof(ZSTD_rust_resetCCtxStorageState)
== offsetof(ZSTD_rust_resetCCtxStorageState, resetExternalSequences)
+ sizeof(void*))
? 1 : -1];
enum {
ZSTD_RUST_RESET_CCTX_RESERVE_ALIGNED64 = 0,
ZSTD_RUST_RESET_CCTX_RESERVE_BUFFER = 1
};
enum {
ZSTD_RUST_RESET_CCTX_POINTER_SEQ_START = 0,
ZSTD_RUST_RESET_CCTX_POINTER_LDM_HASH = 1,
ZSTD_RUST_RESET_CCTX_POINTER_LDM_SEQUENCES = 2,
ZSTD_RUST_RESET_CCTX_POINTER_EXTERNAL_SEQUENCES = 3,
ZSTD_RUST_RESET_CCTX_POINTER_LITERALS = 4,
ZSTD_RUST_RESET_CCTX_POINTER_INPUT_BUFFER = 5,
ZSTD_RUST_RESET_CCTX_POINTER_OUTPUT_BUFFER = 6,
ZSTD_RUST_RESET_CCTX_POINTER_LL_CODE = 7,
ZSTD_RUST_RESET_CCTX_POINTER_ML_CODE = 8,
ZSTD_RUST_RESET_CCTX_POINTER_OF_CODE = 9,
ZSTD_RUST_RESET_CCTX_POINTER_LDM_BUCKETS = 10
};
enum {
ZSTD_RUST_RESET_CCTX_SIZE_MAX_NB_SEQ = 0,
ZSTD_RUST_RESET_CCTX_SIZE_MAX_NB_LIT = 1,
ZSTD_RUST_RESET_CCTX_SIZE_MAX_NB_LDM_SEQ = 2,
ZSTD_RUST_RESET_CCTX_SIZE_EXTERNAL_SEQ_CAPACITY = 3,
ZSTD_RUST_RESET_CCTX_SIZE_INPUT_BUFFER = 4,
ZSTD_RUST_RESET_CCTX_SIZE_OUTPUT_BUFFER = 5
};
enum {
ZSTD_RUST_RESET_CCTX_INT_BUFFERED_POLICY = 0,
ZSTD_RUST_RESET_CCTX_INT_INITIALIZED = 1
};
size_t ZSTD_rust_resetCCtxStorage(const ZSTD_rust_resetCCtxStorageState* state);
size_t ZSTD_rust_estimateCCtxWorkspaceSize(
ZSTD_compressionParameters cParams,
int ldmEnable, U32 ldmHashLog, U32 ldmBucketSizeLog,
@@ -3650,6 +3765,152 @@ static int ZSTD_dictTooBig(size_t const loadedDictSize)
return ZSTD_rust_dictTooBig(loadedDictSize);
}
typedef struct {
ZSTD_CCtx* cctx;
ZSTD_cwksp* ws;
} ZSTD_rust_resetCCtxStorageContext;
static void ZSTD_rust_resetCCtxStorage_setPointer(
void* opaque, int pointerKind, void* pointer)
{
ZSTD_rust_resetCCtxStorageContext* const context =
(ZSTD_rust_resetCCtxStorageContext*)opaque;
ZSTD_CCtx* const cctx = context->cctx;
switch (pointerKind) {
case ZSTD_RUST_RESET_CCTX_POINTER_SEQ_START:
cctx->seqStore.sequencesStart = (SeqDef*)pointer;
break;
case ZSTD_RUST_RESET_CCTX_POINTER_LDM_HASH:
cctx->ldmState.hashTable = (ldmEntry_t*)pointer;
break;
case ZSTD_RUST_RESET_CCTX_POINTER_LDM_SEQUENCES:
cctx->ldmSequences = (rawSeq*)pointer;
break;
case ZSTD_RUST_RESET_CCTX_POINTER_EXTERNAL_SEQUENCES:
cctx->extSeqBuf = (ZSTD_Sequence*)pointer;
break;
case ZSTD_RUST_RESET_CCTX_POINTER_LITERALS:
cctx->seqStore.litStart = (BYTE*)pointer;
break;
case ZSTD_RUST_RESET_CCTX_POINTER_INPUT_BUFFER:
cctx->inBuff = (char*)pointer;
break;
case ZSTD_RUST_RESET_CCTX_POINTER_OUTPUT_BUFFER:
cctx->outBuff = (char*)pointer;
break;
case ZSTD_RUST_RESET_CCTX_POINTER_LL_CODE:
cctx->seqStore.llCode = (BYTE*)pointer;
break;
case ZSTD_RUST_RESET_CCTX_POINTER_ML_CODE:
cctx->seqStore.mlCode = (BYTE*)pointer;
break;
case ZSTD_RUST_RESET_CCTX_POINTER_OF_CODE:
cctx->seqStore.ofCode = (BYTE*)pointer;
break;
case ZSTD_RUST_RESET_CCTX_POINTER_LDM_BUCKETS:
cctx->ldmState.bucketOffsets = (BYTE*)pointer;
break;
default:
assert(0);
break;
}
}
static void ZSTD_rust_resetCCtxStorage_setSize(
void* opaque, int sizeKind, size_t value)
{
ZSTD_rust_resetCCtxStorageContext* const context =
(ZSTD_rust_resetCCtxStorageContext*)opaque;
ZSTD_CCtx* const cctx = context->cctx;
switch (sizeKind) {
case ZSTD_RUST_RESET_CCTX_SIZE_MAX_NB_SEQ:
cctx->seqStore.maxNbSeq = value;
break;
case ZSTD_RUST_RESET_CCTX_SIZE_MAX_NB_LIT:
cctx->seqStore.maxNbLit = value;
break;
case ZSTD_RUST_RESET_CCTX_SIZE_MAX_NB_LDM_SEQ:
cctx->maxNbLdmSequences = value;
break;
case ZSTD_RUST_RESET_CCTX_SIZE_EXTERNAL_SEQ_CAPACITY:
cctx->extSeqBufCapacity = value;
break;
case ZSTD_RUST_RESET_CCTX_SIZE_INPUT_BUFFER:
cctx->inBuffSize = value;
break;
case ZSTD_RUST_RESET_CCTX_SIZE_OUTPUT_BUFFER:
cctx->outBuffSize = value;
break;
default:
assert(0);
break;
}
}
static void ZSTD_rust_resetCCtxStorage_setInt(
void* opaque, int intKind, int value)
{
ZSTD_rust_resetCCtxStorageContext* const context =
(ZSTD_rust_resetCCtxStorageContext*)opaque;
ZSTD_CCtx* const cctx = context->cctx;
switch (intKind) {
case ZSTD_RUST_RESET_CCTX_INT_BUFFERED_POLICY:
cctx->bufferedPolicy = (ZSTD_buffered_policy_e)value;
break;
case ZSTD_RUST_RESET_CCTX_INT_INITIALIZED:
cctx->initialized = value;
break;
default:
assert(0);
break;
}
}
static void* ZSTD_rust_resetCCtxStorage_reserve(
void* opaque, int reserveKind, size_t size)
{
ZSTD_rust_resetCCtxStorageContext* const context =
(ZSTD_rust_resetCCtxStorageContext*)opaque;
switch (reserveKind) {
case ZSTD_RUST_RESET_CCTX_RESERVE_ALIGNED64:
return ZSTD_cwksp_reserve_aligned64(context->ws, size);
case ZSTD_RUST_RESET_CCTX_RESERVE_BUFFER:
return ZSTD_cwksp_reserve_buffer(context->ws, size);
default:
assert(0);
return NULL;
}
}
static int ZSTD_rust_resetCCtxStorage_reserveFailed(void* opaque)
{
ZSTD_rust_resetCCtxStorageContext* const context =
(ZSTD_rust_resetCCtxStorageContext*)opaque;
return ZSTD_cwksp_reserve_failed(context->ws);
}
static void ZSTD_rust_resetCCtxStorage_zero(
void* opaque, void* pointer, size_t size)
{
(void)opaque;
ZSTD_memset(pointer, 0, size);
}
static void ZSTD_rust_resetCCtxStorage_windowInit(void* opaque)
{
ZSTD_rust_resetCCtxStorageContext* const context =
(ZSTD_rust_resetCCtxStorageContext*)opaque;
ZSTD_window_init(&context->cctx->ldmState.window);
context->cctx->ldmState.loadedDictEnd = 0;
}
static void ZSTD_rust_resetCCtxStorage_resetExternalSequences(void* opaque)
{
ZSTD_rust_resetCCtxStorageContext* const context =
(ZSTD_rust_resetCCtxStorageContext*)opaque;
ZSTD_referenceExternalSequences(context->cctx, NULL, 0);
}
/*! ZSTD_resetCCtx_internal() :
* @param loadedDictSize The size of the dictionary to be loaded
* into the context, if any. If no dictionary is used, or the
@@ -3708,6 +3969,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
size_t maxNbLdmSeq;
size_t neededSpace;
ZSTD_indexResetPolicy_e needsIndexReset;
ZSTD_rust_resetCCtxStorageContext storageContext;
ZSTD_rust_resetCCtxStorageState storageState;
resetState.cParams = params->cParams;
resetState.ldmEnable = (int)params->ldmParams.enableLdm;
resetState.ldmHashLog = params->ldmParams.hashLog;
@@ -3737,6 +4000,35 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
maxNbLdmSeq = resetPlan.maxNbLdmSeq;
neededSpace = resetPlan.neededSpace;
needsIndexReset = (ZSTD_indexResetPolicy_e)resetPlan.needsIndexReset;
storageContext.cctx = zc;
storageContext.ws = ws;
storageState.callbackContext = &storageContext;
storageState.ldmEnable = (int)params->ldmParams.enableLdm;
storageState.hasExtSeqProd = ZSTD_hasExtSeqProd(params);
storageState.hashLog = params->ldmParams.hashLog;
storageState.bucketSizeLog = params->ldmParams.bucketSizeLog;
storageState.blockSize = blockSize;
storageState.maxNbSeq = maxNbSeq;
storageState.maxNbLdmSeq = maxNbLdmSeq;
storageState.maxNbExternalSeq = resetPlan.maxNbExternalSeq;
storageState.buffInSize = buffInSize;
storageState.buffOutSize = buffOutSize;
storageState.seqDefSize = sizeof(SeqDef);
storageState.ldmEntrySize = sizeof(ldmEntry_t);
storageState.rawSeqSize = sizeof(rawSeq);
storageState.externalSequenceSize = sizeof(ZSTD_Sequence);
storageState.byteSize = sizeof(BYTE);
storageState.wildcopyOverlength = WILDCOPY_OVERLENGTH;
storageState.bufferedPolicy = (int)zbuff;
storageState.setPointer = ZSTD_rust_resetCCtxStorage_setPointer;
storageState.setSize = ZSTD_rust_resetCCtxStorage_setSize;
storageState.setInt = ZSTD_rust_resetCCtxStorage_setInt;
storageState.reserve = ZSTD_rust_resetCCtxStorage_reserve;
storageState.reserveFailed = ZSTD_rust_resetCCtxStorage_reserveFailed;
storageState.zero = ZSTD_rust_resetCCtxStorage_zero;
storageState.windowInit = ZSTD_rust_resetCCtxStorage_windowInit;
storageState.resetExternalSequences =
ZSTD_rust_resetCCtxStorage_resetExternalSequences;
if (!zc->staticSize) ZSTD_cwksp_bump_oversized_duration(ws, 0);
@@ -3803,65 +4095,11 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
needsIndexReset,
ZSTD_resetTarget_CCtx), "");
zc->seqStore.sequencesStart = (SeqDef*)ZSTD_cwksp_reserve_aligned64(ws, maxNbSeq * sizeof(SeqDef));
/* ldm hash table */
if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
/* TODO: avoid memset? */
size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog;
zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned64(ws, ldmHSize * sizeof(ldmEntry_t));
ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t));
zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned64(ws, maxNbLdmSeq * sizeof(rawSeq));
zc->maxNbLdmSequences = maxNbLdmSeq;
ZSTD_window_init(&zc->ldmState.window);
zc->ldmState.loadedDictEnd = 0;
}
/* reserve space for block-level external sequences */
if (ZSTD_hasExtSeqProd(params)) {
size_t const maxNbExternalSeq = resetPlan.maxNbExternalSeq;
zc->extSeqBufCapacity = maxNbExternalSeq;
zc->extSeqBuf =
(ZSTD_Sequence*)ZSTD_cwksp_reserve_aligned64(ws, maxNbExternalSeq * sizeof(ZSTD_Sequence));
}
/* buffers */
/* ZSTD_wildcopy() is used to copy into the literals buffer,
* so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes.
*/
zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH);
zc->seqStore.maxNbLit = blockSize;
zc->bufferedPolicy = zbuff;
zc->inBuffSize = buffInSize;
zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize);
zc->outBuffSize = buffOutSize;
zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize);
/* ldm bucketOffsets table */
if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
/* TODO: avoid memset? */
size_t const numBuckets =
((size_t)1) << (params->ldmParams.hashLog -
params->ldmParams.bucketSizeLog);
zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, numBuckets);
ZSTD_memset(zc->ldmState.bucketOffsets, 0, numBuckets);
}
/* sequences storage */
ZSTD_referenceExternalSequences(zc, NULL, 0);
zc->seqStore.maxNbSeq = maxNbSeq;
zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
FORWARD_IF_ERROR(ZSTD_rust_resetCCtxStorage(&storageState), "");
DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws));
assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace));
zc->initialized = 1;
return 0;
}
}