feat(compress): move CCtx workspace policy into Rust
Move the CCtx reset workspace resize decision, index-reset mutation, reservation order, private-pointer publication order, and clear boundary into Rust. Keep the workspace allocator and private ZSTD_CCtx layout operations in C behind callbacks, with matching C/Rust layout assertions. Add focused tests for the no-resize, resize, and static-resize error paths so the callback order and mutation boundaries remain explicit. Test Plan: - `ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml` - `ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings` - `ulimit -v 41943040 && make -j1` - `ulimit -v 41943040 && make -j1 -C tests test-zstream ZSTREAM_TESTTIME=-T2s` - `ulimit -v 41943040 && make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests` - `ulimit -v 41943040 && cargo fmt --manifest-path rust/Cargo.toml -- --check` - `git diff --check`
This commit is contained in:
+120
-38
@@ -1443,6 +1443,10 @@ typedef int (*ZSTD_rust_resetCCtxStorageReserveFailed_f)(void* context);
|
|||||||
typedef void (*ZSTD_rust_resetCCtxStorageZero_f)(
|
typedef void (*ZSTD_rust_resetCCtxStorageZero_f)(
|
||||||
void* context, void* pointer, size_t size);
|
void* context, void* pointer, size_t size);
|
||||||
typedef void (*ZSTD_rust_resetCCtxStorageCallback_f)(void* context);
|
typedef void (*ZSTD_rust_resetCCtxStorageCallback_f)(void* context);
|
||||||
|
typedef size_t (*ZSTD_rust_resetCCtxWorkspaceCreate_f)(
|
||||||
|
void* context, size_t neededSpace);
|
||||||
|
typedef void* (*ZSTD_rust_resetCCtxWorkspaceReserveObject_f)(
|
||||||
|
void* context, size_t size);
|
||||||
typedef struct {
|
typedef struct {
|
||||||
void* callbackContext;
|
void* callbackContext;
|
||||||
int ldmEnable;
|
int ldmEnable;
|
||||||
@@ -1516,6 +1520,45 @@ typedef char ZSTD_rust_reset_cctx_storage_state_layout[
|
|||||||
== offsetof(ZSTD_rust_resetCCtxStorageState, resetExternalSequences)
|
== offsetof(ZSTD_rust_resetCCtxStorageState, resetExternalSequences)
|
||||||
+ sizeof(void*))
|
+ sizeof(void*))
|
||||||
? 1 : -1];
|
? 1 : -1];
|
||||||
|
typedef struct {
|
||||||
|
void* callbackContext;
|
||||||
|
int isStatic;
|
||||||
|
int workspaceTooSmall;
|
||||||
|
int workspaceWasteful;
|
||||||
|
size_t neededSpace;
|
||||||
|
size_t compressedBlockStateSize;
|
||||||
|
size_t tmpWorkspaceSize;
|
||||||
|
int* needsIndexReset;
|
||||||
|
ZSTD_rust_resetCCtxStorageCallback_f bumpOversizedDuration;
|
||||||
|
ZSTD_rust_resetCCtxStorageCallback_f freeWorkspace;
|
||||||
|
ZSTD_rust_resetCCtxWorkspaceCreate_f createWorkspace;
|
||||||
|
ZSTD_rust_resetCCtxWorkspaceReserveObject_f reserveObject;
|
||||||
|
ZSTD_rust_resetCCtxStorageSetPointer_f setPointer;
|
||||||
|
ZSTD_rust_resetCCtxStorageSetSize_f setSize;
|
||||||
|
ZSTD_rust_resetCCtxStorageCallback_f clearWorkspace;
|
||||||
|
} ZSTD_rust_resetCCtxWorkspaceState;
|
||||||
|
typedef char ZSTD_rust_reset_cctx_workspace_state_layout[
|
||||||
|
(offsetof(ZSTD_rust_resetCCtxWorkspaceState, callbackContext) == 0
|
||||||
|
&& offsetof(ZSTD_rust_resetCCtxWorkspaceState, isStatic) == sizeof(void*)
|
||||||
|
&& offsetof(ZSTD_rust_resetCCtxWorkspaceState, workspaceTooSmall)
|
||||||
|
== sizeof(void*) + sizeof(int)
|
||||||
|
&& offsetof(ZSTD_rust_resetCCtxWorkspaceState, workspaceWasteful)
|
||||||
|
== sizeof(void*) + 2 * sizeof(int)
|
||||||
|
&& offsetof(ZSTD_rust_resetCCtxWorkspaceState, neededSpace)
|
||||||
|
> offsetof(ZSTD_rust_resetCCtxWorkspaceState, workspaceWasteful)
|
||||||
|
&& offsetof(ZSTD_rust_resetCCtxWorkspaceState, compressedBlockStateSize)
|
||||||
|
== offsetof(ZSTD_rust_resetCCtxWorkspaceState, neededSpace)
|
||||||
|
+ sizeof(size_t)
|
||||||
|
&& offsetof(ZSTD_rust_resetCCtxWorkspaceState, tmpWorkspaceSize)
|
||||||
|
== offsetof(ZSTD_rust_resetCCtxWorkspaceState, compressedBlockStateSize)
|
||||||
|
+ sizeof(size_t)
|
||||||
|
&& offsetof(ZSTD_rust_resetCCtxWorkspaceState, needsIndexReset)
|
||||||
|
== offsetof(ZSTD_rust_resetCCtxWorkspaceState, tmpWorkspaceSize)
|
||||||
|
+ sizeof(size_t)
|
||||||
|
&& sizeof(ZSTD_rust_resetCCtxWorkspaceState)
|
||||||
|
== offsetof(ZSTD_rust_resetCCtxWorkspaceState, clearWorkspace)
|
||||||
|
+ sizeof(void*))
|
||||||
|
? 1 : -1];
|
||||||
enum {
|
enum {
|
||||||
ZSTD_RUST_RESET_CCTX_RESERVE_ALIGNED64 = 0,
|
ZSTD_RUST_RESET_CCTX_RESERVE_ALIGNED64 = 0,
|
||||||
ZSTD_RUST_RESET_CCTX_RESERVE_BUFFER = 1
|
ZSTD_RUST_RESET_CCTX_RESERVE_BUFFER = 1
|
||||||
@@ -1531,7 +1574,10 @@ enum {
|
|||||||
ZSTD_RUST_RESET_CCTX_POINTER_LL_CODE = 7,
|
ZSTD_RUST_RESET_CCTX_POINTER_LL_CODE = 7,
|
||||||
ZSTD_RUST_RESET_CCTX_POINTER_ML_CODE = 8,
|
ZSTD_RUST_RESET_CCTX_POINTER_ML_CODE = 8,
|
||||||
ZSTD_RUST_RESET_CCTX_POINTER_OF_CODE = 9,
|
ZSTD_RUST_RESET_CCTX_POINTER_OF_CODE = 9,
|
||||||
ZSTD_RUST_RESET_CCTX_POINTER_LDM_BUCKETS = 10
|
ZSTD_RUST_RESET_CCTX_POINTER_LDM_BUCKETS = 10,
|
||||||
|
ZSTD_RUST_RESET_CCTX_POINTER_PREV_CBLOCK = 11,
|
||||||
|
ZSTD_RUST_RESET_CCTX_POINTER_NEXT_CBLOCK = 12,
|
||||||
|
ZSTD_RUST_RESET_CCTX_POINTER_TMP_WORKSPACE = 13
|
||||||
};
|
};
|
||||||
enum {
|
enum {
|
||||||
ZSTD_RUST_RESET_CCTX_SIZE_MAX_NB_SEQ = 0,
|
ZSTD_RUST_RESET_CCTX_SIZE_MAX_NB_SEQ = 0,
|
||||||
@@ -1539,13 +1585,16 @@ enum {
|
|||||||
ZSTD_RUST_RESET_CCTX_SIZE_MAX_NB_LDM_SEQ = 2,
|
ZSTD_RUST_RESET_CCTX_SIZE_MAX_NB_LDM_SEQ = 2,
|
||||||
ZSTD_RUST_RESET_CCTX_SIZE_EXTERNAL_SEQ_CAPACITY = 3,
|
ZSTD_RUST_RESET_CCTX_SIZE_EXTERNAL_SEQ_CAPACITY = 3,
|
||||||
ZSTD_RUST_RESET_CCTX_SIZE_INPUT_BUFFER = 4,
|
ZSTD_RUST_RESET_CCTX_SIZE_INPUT_BUFFER = 4,
|
||||||
ZSTD_RUST_RESET_CCTX_SIZE_OUTPUT_BUFFER = 5
|
ZSTD_RUST_RESET_CCTX_SIZE_OUTPUT_BUFFER = 5,
|
||||||
|
ZSTD_RUST_RESET_CCTX_SIZE_TMP_WORKSPACE = 6
|
||||||
};
|
};
|
||||||
enum {
|
enum {
|
||||||
ZSTD_RUST_RESET_CCTX_INT_BUFFERED_POLICY = 0,
|
ZSTD_RUST_RESET_CCTX_INT_BUFFERED_POLICY = 0,
|
||||||
ZSTD_RUST_RESET_CCTX_INT_INITIALIZED = 1
|
ZSTD_RUST_RESET_CCTX_INT_INITIALIZED = 1
|
||||||
};
|
};
|
||||||
size_t ZSTD_rust_resetCCtxStorage(const ZSTD_rust_resetCCtxStorageState* state);
|
size_t ZSTD_rust_resetCCtxStorage(const ZSTD_rust_resetCCtxStorageState* state);
|
||||||
|
size_t ZSTD_rust_resetCCtxWorkspace(
|
||||||
|
const ZSTD_rust_resetCCtxWorkspaceState* state);
|
||||||
size_t ZSTD_rust_estimateCCtxWorkspaceSize(
|
size_t ZSTD_rust_estimateCCtxWorkspaceSize(
|
||||||
ZSTD_compressionParameters cParams,
|
ZSTD_compressionParameters cParams,
|
||||||
int ldmEnable, U32 ldmHashLog, U32 ldmBucketSizeLog,
|
int ldmEnable, U32 ldmHashLog, U32 ldmBucketSizeLog,
|
||||||
@@ -4028,6 +4077,15 @@ static void ZSTD_rust_resetCCtxStorage_setPointer(
|
|||||||
case ZSTD_RUST_RESET_CCTX_POINTER_LDM_BUCKETS:
|
case ZSTD_RUST_RESET_CCTX_POINTER_LDM_BUCKETS:
|
||||||
cctx->ldmState.bucketOffsets = (BYTE*)pointer;
|
cctx->ldmState.bucketOffsets = (BYTE*)pointer;
|
||||||
break;
|
break;
|
||||||
|
case ZSTD_RUST_RESET_CCTX_POINTER_PREV_CBLOCK:
|
||||||
|
cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)pointer;
|
||||||
|
break;
|
||||||
|
case ZSTD_RUST_RESET_CCTX_POINTER_NEXT_CBLOCK:
|
||||||
|
cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)pointer;
|
||||||
|
break;
|
||||||
|
case ZSTD_RUST_RESET_CCTX_POINTER_TMP_WORKSPACE:
|
||||||
|
cctx->tmpWorkspace = pointer;
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
assert(0);
|
assert(0);
|
||||||
break;
|
break;
|
||||||
@@ -4059,6 +4117,9 @@ static void ZSTD_rust_resetCCtxStorage_setSize(
|
|||||||
case ZSTD_RUST_RESET_CCTX_SIZE_OUTPUT_BUFFER:
|
case ZSTD_RUST_RESET_CCTX_SIZE_OUTPUT_BUFFER:
|
||||||
cctx->outBuffSize = value;
|
cctx->outBuffSize = value;
|
||||||
break;
|
break;
|
||||||
|
case ZSTD_RUST_RESET_CCTX_SIZE_TMP_WORKSPACE:
|
||||||
|
cctx->tmpWkspSize = value;
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
assert(0);
|
assert(0);
|
||||||
break;
|
break;
|
||||||
@@ -4129,6 +4190,44 @@ static void ZSTD_rust_resetCCtxStorage_resetExternalSequences(void* opaque)
|
|||||||
ZSTD_referenceExternalSequences(context->cctx, NULL, 0);
|
ZSTD_referenceExternalSequences(context->cctx, NULL, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void ZSTD_rust_resetCCtxWorkspace_bumpOversizedDuration(void* opaque)
|
||||||
|
{
|
||||||
|
ZSTD_rust_resetCCtxStorageContext* const context =
|
||||||
|
(ZSTD_rust_resetCCtxStorageContext*)opaque;
|
||||||
|
ZSTD_cwksp_bump_oversized_duration(context->ws, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void ZSTD_rust_resetCCtxWorkspace_free(void* opaque)
|
||||||
|
{
|
||||||
|
ZSTD_rust_resetCCtxStorageContext* const context =
|
||||||
|
(ZSTD_rust_resetCCtxStorageContext*)opaque;
|
||||||
|
ZSTD_cwksp_free(context->ws, context->cctx->customMem);
|
||||||
|
}
|
||||||
|
|
||||||
|
static size_t ZSTD_rust_resetCCtxWorkspace_create(
|
||||||
|
void* opaque, size_t neededSpace)
|
||||||
|
{
|
||||||
|
ZSTD_rust_resetCCtxStorageContext* const context =
|
||||||
|
(ZSTD_rust_resetCCtxStorageContext*)opaque;
|
||||||
|
return ZSTD_cwksp_create(
|
||||||
|
context->ws, neededSpace, context->cctx->customMem);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void* ZSTD_rust_resetCCtxWorkspace_reserveObject(
|
||||||
|
void* opaque, size_t size)
|
||||||
|
{
|
||||||
|
ZSTD_rust_resetCCtxStorageContext* const context =
|
||||||
|
(ZSTD_rust_resetCCtxStorageContext*)opaque;
|
||||||
|
return ZSTD_cwksp_reserve_object(context->ws, size);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void ZSTD_rust_resetCCtxWorkspace_clear(void* opaque)
|
||||||
|
{
|
||||||
|
ZSTD_rust_resetCCtxStorageContext* const context =
|
||||||
|
(ZSTD_rust_resetCCtxStorageContext*)opaque;
|
||||||
|
ZSTD_cwksp_clear(context->ws);
|
||||||
|
}
|
||||||
|
|
||||||
/*! ZSTD_resetCCtx_internal() :
|
/*! ZSTD_resetCCtx_internal() :
|
||||||
* @param loadedDictSize The size of the dictionary to be loaded
|
* @param loadedDictSize The size of the dictionary to be loaded
|
||||||
* into the context, if any. If no dictionary is used, or the
|
* into the context, if any. If no dictionary is used, or the
|
||||||
@@ -4248,42 +4347,25 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
|
|||||||
storageState.resetExternalSequences =
|
storageState.resetExternalSequences =
|
||||||
ZSTD_rust_resetCCtxStorage_resetExternalSequences;
|
ZSTD_rust_resetCCtxStorage_resetExternalSequences;
|
||||||
|
|
||||||
if (!zc->staticSize) ZSTD_cwksp_bump_oversized_duration(ws, 0);
|
{ ZSTD_rust_resetCCtxWorkspaceState workspaceState;
|
||||||
|
workspaceState.callbackContext = &storageContext;
|
||||||
{ /* Check if workspace is large enough, alloc a new one if needed */
|
workspaceState.isStatic = zc->staticSize != 0;
|
||||||
int const workspaceTooSmall = ZSTD_cwksp_sizeof(ws) < neededSpace;
|
workspaceState.workspaceTooSmall = ZSTD_cwksp_sizeof(ws) < neededSpace;
|
||||||
int const workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace);
|
workspaceState.workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace);
|
||||||
int resizeWorkspace = workspaceTooSmall || workspaceWasteful;
|
workspaceState.neededSpace = neededSpace;
|
||||||
DEBUGLOG(4, "Need %zu B workspace", neededSpace);
|
workspaceState.compressedBlockStateSize = sizeof(ZSTD_compressedBlockState_t);
|
||||||
DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", resetPlan.windowSize, blockSize);
|
workspaceState.tmpWorkspaceSize = TMP_WORKSPACE_SIZE;
|
||||||
|
workspaceState.needsIndexReset = (int*)&needsIndexReset;
|
||||||
if (resizeWorkspace) {
|
workspaceState.bumpOversizedDuration =
|
||||||
DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB",
|
ZSTD_rust_resetCCtxWorkspace_bumpOversizedDuration;
|
||||||
ZSTD_cwksp_sizeof(ws) >> 10,
|
workspaceState.freeWorkspace = ZSTD_rust_resetCCtxWorkspace_free;
|
||||||
neededSpace >> 10);
|
workspaceState.createWorkspace = ZSTD_rust_resetCCtxWorkspace_create;
|
||||||
|
workspaceState.reserveObject = ZSTD_rust_resetCCtxWorkspace_reserveObject;
|
||||||
RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize");
|
workspaceState.setPointer = ZSTD_rust_resetCCtxStorage_setPointer;
|
||||||
|
workspaceState.setSize = ZSTD_rust_resetCCtxStorage_setSize;
|
||||||
needsIndexReset = ZSTDirp_reset;
|
workspaceState.clearWorkspace = ZSTD_rust_resetCCtxWorkspace_clear;
|
||||||
|
FORWARD_IF_ERROR(ZSTD_rust_resetCCtxWorkspace(&workspaceState), "");
|
||||||
ZSTD_cwksp_free(ws, zc->customMem);
|
}
|
||||||
FORWARD_IF_ERROR(ZSTD_cwksp_create(ws, neededSpace, zc->customMem), "");
|
|
||||||
|
|
||||||
DEBUGLOG(5, "reserving object space");
|
|
||||||
/* Statically sized space.
|
|
||||||
* tmpWorkspace never moves,
|
|
||||||
* though prev/next block swap places */
|
|
||||||
assert(ZSTD_cwksp_check_available(ws, 2 * sizeof(ZSTD_compressedBlockState_t)));
|
|
||||||
zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t));
|
|
||||||
RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock");
|
|
||||||
zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t));
|
|
||||||
RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock");
|
|
||||||
zc->tmpWorkspace = ZSTD_cwksp_reserve_object(ws, TMP_WORKSPACE_SIZE);
|
|
||||||
RETURN_ERROR_IF(zc->tmpWorkspace == NULL, memory_allocation, "couldn't allocate tmpWorkspace");
|
|
||||||
zc->tmpWkspSize = TMP_WORKSPACE_SIZE;
|
|
||||||
} }
|
|
||||||
|
|
||||||
ZSTD_cwksp_clear(ws);
|
|
||||||
|
|
||||||
/* init params */
|
/* init params */
|
||||||
zc->blockState.matchState.cParams = params->cParams;
|
zc->blockState.matchState.cParams = params->cParams;
|
||||||
|
|||||||
@@ -6574,6 +6574,9 @@ const RESET_CCTX_POINTER_LL_CODE: c_int = 7;
|
|||||||
const RESET_CCTX_POINTER_ML_CODE: c_int = 8;
|
const RESET_CCTX_POINTER_ML_CODE: c_int = 8;
|
||||||
const RESET_CCTX_POINTER_OF_CODE: c_int = 9;
|
const RESET_CCTX_POINTER_OF_CODE: c_int = 9;
|
||||||
const RESET_CCTX_POINTER_LDM_BUCKETS: c_int = 10;
|
const RESET_CCTX_POINTER_LDM_BUCKETS: c_int = 10;
|
||||||
|
const RESET_CCTX_POINTER_PREV_CBLOCK: c_int = 11;
|
||||||
|
const RESET_CCTX_POINTER_NEXT_CBLOCK: c_int = 12;
|
||||||
|
const RESET_CCTX_POINTER_TMP_WORKSPACE: c_int = 13;
|
||||||
|
|
||||||
const RESET_CCTX_SIZE_MAX_NB_SEQ: c_int = 0;
|
const RESET_CCTX_SIZE_MAX_NB_SEQ: c_int = 0;
|
||||||
const RESET_CCTX_SIZE_MAX_NB_LIT: c_int = 1;
|
const RESET_CCTX_SIZE_MAX_NB_LIT: c_int = 1;
|
||||||
@@ -6581,9 +6584,11 @@ const RESET_CCTX_SIZE_MAX_NB_LDM_SEQ: c_int = 2;
|
|||||||
const RESET_CCTX_SIZE_EXTERNAL_SEQ_CAPACITY: c_int = 3;
|
const RESET_CCTX_SIZE_EXTERNAL_SEQ_CAPACITY: c_int = 3;
|
||||||
const RESET_CCTX_SIZE_INPUT_BUFFER: c_int = 4;
|
const RESET_CCTX_SIZE_INPUT_BUFFER: c_int = 4;
|
||||||
const RESET_CCTX_SIZE_OUTPUT_BUFFER: c_int = 5;
|
const RESET_CCTX_SIZE_OUTPUT_BUFFER: c_int = 5;
|
||||||
|
const RESET_CCTX_SIZE_TMP_WORKSPACE: c_int = 6;
|
||||||
|
|
||||||
const RESET_CCTX_INT_BUFFERED_POLICY: c_int = 0;
|
const RESET_CCTX_INT_BUFFERED_POLICY: c_int = 0;
|
||||||
const RESET_CCTX_INT_INITIALIZED: c_int = 1;
|
const RESET_CCTX_INT_INITIALIZED: c_int = 1;
|
||||||
|
const RESET_CCTX_INDEX_RESET: c_int = 1;
|
||||||
|
|
||||||
/// C-owned workspace/layout projection for the post-match-state reset tail.
|
/// C-owned workspace/layout projection for the post-match-state reset tail.
|
||||||
///
|
///
|
||||||
@@ -6713,6 +6718,88 @@ const _: () = {
|
|||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type ResetCCtxWorkspaceCreate = unsafe extern "C" fn(*mut c_void, usize) -> usize;
|
||||||
|
type ResetCCtxWorkspaceReserveObject = unsafe extern "C" fn(*mut c_void, usize) -> *mut c_void;
|
||||||
|
|
||||||
|
/// C-owned workspace operations used by the CCtx reset policy.
|
||||||
|
///
|
||||||
|
/// Rust owns the resize decision, error ordering, object-reservation order,
|
||||||
|
/// and workspace-clear boundary. C retains the workspace representation,
|
||||||
|
/// allocator, and private CCtx publication callbacks.
|
||||||
|
#[repr(C)]
|
||||||
|
pub struct ZSTD_rust_resetCCtxWorkspaceState {
|
||||||
|
callbackContext: *mut c_void,
|
||||||
|
isStatic: c_int,
|
||||||
|
workspaceTooSmall: c_int,
|
||||||
|
workspaceWasteful: c_int,
|
||||||
|
neededSpace: usize,
|
||||||
|
compressedBlockStateSize: usize,
|
||||||
|
tmpWorkspaceSize: usize,
|
||||||
|
needsIndexReset: *mut c_int,
|
||||||
|
bumpOversizedDuration: Option<ResetCCtxStorageCallback>,
|
||||||
|
freeWorkspace: Option<ResetCCtxStorageCallback>,
|
||||||
|
createWorkspace: Option<ResetCCtxWorkspaceCreate>,
|
||||||
|
reserveObject: Option<ResetCCtxWorkspaceReserveObject>,
|
||||||
|
setPointer: Option<ResetCCtxStorageSetPointer>,
|
||||||
|
setSize: Option<ResetCCtxStorageSetSize>,
|
||||||
|
clearWorkspace: Option<ResetCCtxStorageCallback>,
|
||||||
|
}
|
||||||
|
|
||||||
|
const _: () = {
|
||||||
|
assert!(offset_of!(ZSTD_rust_resetCCtxWorkspaceState, callbackContext) == 0);
|
||||||
|
assert!(offset_of!(ZSTD_rust_resetCCtxWorkspaceState, isStatic) == size_of::<usize>());
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_resetCCtxWorkspaceState, workspaceTooSmall)
|
||||||
|
== size_of::<usize>() + size_of::<c_int>()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_resetCCtxWorkspaceState, workspaceWasteful)
|
||||||
|
== size_of::<usize>() + 2 * size_of::<c_int>()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_resetCCtxWorkspaceState, neededSpace)
|
||||||
|
> offset_of!(ZSTD_rust_resetCCtxWorkspaceState, workspaceWasteful)
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_resetCCtxWorkspaceState, compressedBlockStateSize)
|
||||||
|
== offset_of!(ZSTD_rust_resetCCtxWorkspaceState, neededSpace) + size_of::<usize>()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_resetCCtxWorkspaceState, tmpWorkspaceSize)
|
||||||
|
== offset_of!(ZSTD_rust_resetCCtxWorkspaceState, compressedBlockStateSize)
|
||||||
|
+ size_of::<usize>()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_resetCCtxWorkspaceState, needsIndexReset)
|
||||||
|
== offset_of!(ZSTD_rust_resetCCtxWorkspaceState, tmpWorkspaceSize) + size_of::<usize>()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_resetCCtxWorkspaceState, freeWorkspace)
|
||||||
|
== offset_of!(ZSTD_rust_resetCCtxWorkspaceState, bumpOversizedDuration)
|
||||||
|
+ size_of::<usize>()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_resetCCtxWorkspaceState, createWorkspace)
|
||||||
|
== offset_of!(ZSTD_rust_resetCCtxWorkspaceState, freeWorkspace) + size_of::<usize>()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_resetCCtxWorkspaceState, reserveObject)
|
||||||
|
== offset_of!(ZSTD_rust_resetCCtxWorkspaceState, createWorkspace) + size_of::<usize>()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_resetCCtxWorkspaceState, setPointer)
|
||||||
|
== offset_of!(ZSTD_rust_resetCCtxWorkspaceState, reserveObject) + size_of::<usize>()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_resetCCtxWorkspaceState, setSize)
|
||||||
|
== offset_of!(ZSTD_rust_resetCCtxWorkspaceState, setPointer) + size_of::<usize>()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
size_of::<ZSTD_rust_resetCCtxWorkspaceState>()
|
||||||
|
== offset_of!(ZSTD_rust_resetCCtxWorkspaceState, clearWorkspace) + size_of::<usize>()
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
/// Reserve and publish the private CCtx storage that follows match-state reset.
|
/// Reserve and publish the private CCtx storage that follows match-state reset.
|
||||||
#[no_mangle]
|
#[no_mangle]
|
||||||
pub unsafe extern "C" fn ZSTD_rust_resetCCtxStorage(
|
pub unsafe extern "C" fn ZSTD_rust_resetCCtxStorage(
|
||||||
@@ -6909,6 +6996,92 @@ pub unsafe extern "C" fn ZSTD_rust_resetCCtxStorage(
|
|||||||
0
|
0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Own the CCtx workspace resize, object reservation, and clear order.
|
||||||
|
#[no_mangle]
|
||||||
|
pub unsafe extern "C" fn ZSTD_rust_resetCCtxWorkspace(
|
||||||
|
state: *const ZSTD_rust_resetCCtxWorkspaceState,
|
||||||
|
) -> usize {
|
||||||
|
if state.is_null() {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
}
|
||||||
|
let state = unsafe { &*state };
|
||||||
|
if state.callbackContext.is_null() || state.needsIndexReset.is_null() {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
}
|
||||||
|
let Some(bump_oversized_duration) = state.bumpOversizedDuration else {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
};
|
||||||
|
let Some(free_workspace) = state.freeWorkspace else {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
};
|
||||||
|
let Some(create_workspace) = state.createWorkspace else {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
};
|
||||||
|
let Some(reserve_object) = state.reserveObject else {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
};
|
||||||
|
let Some(set_pointer) = state.setPointer else {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
};
|
||||||
|
let Some(set_size) = state.setSize else {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
};
|
||||||
|
let Some(clear_workspace) = state.clearWorkspace else {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
};
|
||||||
|
|
||||||
|
unsafe {
|
||||||
|
if state.isStatic == 0 {
|
||||||
|
bump_oversized_duration(state.callbackContext);
|
||||||
|
}
|
||||||
|
let resize_workspace = state.workspaceTooSmall != 0 || state.workspaceWasteful != 0;
|
||||||
|
if resize_workspace {
|
||||||
|
if state.isStatic != 0 {
|
||||||
|
return ERROR(ZstdErrorCode::MemoryAllocation);
|
||||||
|
}
|
||||||
|
*state.needsIndexReset = RESET_CCTX_INDEX_RESET;
|
||||||
|
free_workspace(state.callbackContext);
|
||||||
|
let result = create_workspace(state.callbackContext, state.neededSpace);
|
||||||
|
if ERR_isError(result) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
let previous = reserve_object(state.callbackContext, state.compressedBlockStateSize);
|
||||||
|
if previous.is_null() {
|
||||||
|
return ERROR(ZstdErrorCode::MemoryAllocation);
|
||||||
|
}
|
||||||
|
set_pointer(
|
||||||
|
state.callbackContext,
|
||||||
|
RESET_CCTX_POINTER_PREV_CBLOCK,
|
||||||
|
previous,
|
||||||
|
);
|
||||||
|
|
||||||
|
let next = reserve_object(state.callbackContext, state.compressedBlockStateSize);
|
||||||
|
if next.is_null() {
|
||||||
|
return ERROR(ZstdErrorCode::MemoryAllocation);
|
||||||
|
}
|
||||||
|
set_pointer(state.callbackContext, RESET_CCTX_POINTER_NEXT_CBLOCK, next);
|
||||||
|
|
||||||
|
let tmp_workspace = reserve_object(state.callbackContext, state.tmpWorkspaceSize);
|
||||||
|
if tmp_workspace.is_null() {
|
||||||
|
return ERROR(ZstdErrorCode::MemoryAllocation);
|
||||||
|
}
|
||||||
|
set_pointer(
|
||||||
|
state.callbackContext,
|
||||||
|
RESET_CCTX_POINTER_TMP_WORKSPACE,
|
||||||
|
tmp_workspace,
|
||||||
|
);
|
||||||
|
set_size(
|
||||||
|
state.callbackContext,
|
||||||
|
RESET_CCTX_SIZE_TMP_WORKSPACE,
|
||||||
|
state.tmpWorkspaceSize,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
clear_workspace(state.callbackContext);
|
||||||
|
}
|
||||||
|
0
|
||||||
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
fn max_estimate_cctx_size(
|
fn max_estimate_cctx_size(
|
||||||
estimate0: usize,
|
estimate0: usize,
|
||||||
@@ -16276,6 +16449,210 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, PartialEq)]
|
||||||
|
enum ResetCCtxWorkspaceTestEvent {
|
||||||
|
BumpOversizedDuration,
|
||||||
|
FreeWorkspace,
|
||||||
|
CreateWorkspace(usize),
|
||||||
|
ReserveObject(usize),
|
||||||
|
Pointer(c_int),
|
||||||
|
Size(c_int, usize),
|
||||||
|
ClearWorkspace,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct ResetCCtxWorkspaceTestContext {
|
||||||
|
events: Vec<ResetCCtxWorkspaceTestEvent>,
|
||||||
|
reserve_count: usize,
|
||||||
|
create_result: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe fn reset_cctx_workspace_test_context(
|
||||||
|
context: *mut c_void,
|
||||||
|
) -> &'static mut ResetCCtxWorkspaceTestContext {
|
||||||
|
unsafe { &mut *context.cast::<ResetCCtxWorkspaceTestContext>() }
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe extern "C" fn reset_cctx_workspace_test_bump(context: *mut c_void) {
|
||||||
|
let context = unsafe { reset_cctx_workspace_test_context(context) };
|
||||||
|
context
|
||||||
|
.events
|
||||||
|
.push(ResetCCtxWorkspaceTestEvent::BumpOversizedDuration);
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe extern "C" fn reset_cctx_workspace_test_free(context: *mut c_void) {
|
||||||
|
let context = unsafe { reset_cctx_workspace_test_context(context) };
|
||||||
|
context
|
||||||
|
.events
|
||||||
|
.push(ResetCCtxWorkspaceTestEvent::FreeWorkspace);
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe extern "C" fn reset_cctx_workspace_test_create(
|
||||||
|
context: *mut c_void,
|
||||||
|
needed_space: usize,
|
||||||
|
) -> usize {
|
||||||
|
let context = unsafe { reset_cctx_workspace_test_context(context) };
|
||||||
|
context
|
||||||
|
.events
|
||||||
|
.push(ResetCCtxWorkspaceTestEvent::CreateWorkspace(needed_space));
|
||||||
|
context.create_result
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe extern "C" fn reset_cctx_workspace_test_reserve_object(
|
||||||
|
context: *mut c_void,
|
||||||
|
size: usize,
|
||||||
|
) -> *mut c_void {
|
||||||
|
let context = unsafe { reset_cctx_workspace_test_context(context) };
|
||||||
|
context
|
||||||
|
.events
|
||||||
|
.push(ResetCCtxWorkspaceTestEvent::ReserveObject(size));
|
||||||
|
let pointer = (0x1000usize + context.reserve_count * 0x1000) as *mut c_void;
|
||||||
|
context.reserve_count += 1;
|
||||||
|
pointer
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe extern "C" fn reset_cctx_workspace_test_set_pointer(
|
||||||
|
context: *mut c_void,
|
||||||
|
kind: c_int,
|
||||||
|
_pointer: *mut c_void,
|
||||||
|
) {
|
||||||
|
let context = unsafe { reset_cctx_workspace_test_context(context) };
|
||||||
|
context
|
||||||
|
.events
|
||||||
|
.push(ResetCCtxWorkspaceTestEvent::Pointer(kind));
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe extern "C" fn reset_cctx_workspace_test_set_size(
|
||||||
|
context: *mut c_void,
|
||||||
|
kind: c_int,
|
||||||
|
value: usize,
|
||||||
|
) {
|
||||||
|
let context = unsafe { reset_cctx_workspace_test_context(context) };
|
||||||
|
context
|
||||||
|
.events
|
||||||
|
.push(ResetCCtxWorkspaceTestEvent::Size(kind, value));
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe extern "C" fn reset_cctx_workspace_test_clear(context: *mut c_void) {
|
||||||
|
let context = unsafe { reset_cctx_workspace_test_context(context) };
|
||||||
|
context
|
||||||
|
.events
|
||||||
|
.push(ResetCCtxWorkspaceTestEvent::ClearWorkspace);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reset_cctx_workspace_test_state(
|
||||||
|
context: &mut ResetCCtxWorkspaceTestContext,
|
||||||
|
is_static: c_int,
|
||||||
|
workspace_too_small: c_int,
|
||||||
|
workspace_wasteful: c_int,
|
||||||
|
needed_space: usize,
|
||||||
|
compressed_block_state_size: usize,
|
||||||
|
tmp_workspace_size: usize,
|
||||||
|
needs_index_reset: &mut c_int,
|
||||||
|
) -> ZSTD_rust_resetCCtxWorkspaceState {
|
||||||
|
ZSTD_rust_resetCCtxWorkspaceState {
|
||||||
|
callbackContext: (context as *mut ResetCCtxWorkspaceTestContext).cast(),
|
||||||
|
isStatic: is_static,
|
||||||
|
workspaceTooSmall: workspace_too_small,
|
||||||
|
workspaceWasteful: workspace_wasteful,
|
||||||
|
neededSpace: needed_space,
|
||||||
|
compressedBlockStateSize: compressed_block_state_size,
|
||||||
|
tmpWorkspaceSize: tmp_workspace_size,
|
||||||
|
needsIndexReset: needs_index_reset,
|
||||||
|
bumpOversizedDuration: Some(reset_cctx_workspace_test_bump),
|
||||||
|
freeWorkspace: Some(reset_cctx_workspace_test_free),
|
||||||
|
createWorkspace: Some(reset_cctx_workspace_test_create),
|
||||||
|
reserveObject: Some(reset_cctx_workspace_test_reserve_object),
|
||||||
|
setPointer: Some(reset_cctx_workspace_test_set_pointer),
|
||||||
|
setSize: Some(reset_cctx_workspace_test_set_size),
|
||||||
|
clearWorkspace: Some(reset_cctx_workspace_test_clear),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cctx_workspace_preserves_existing_workspace_and_clear_order() {
|
||||||
|
let mut context = ResetCCtxWorkspaceTestContext::default();
|
||||||
|
let mut needs_index_reset = 0;
|
||||||
|
let state = reset_cctx_workspace_test_state(
|
||||||
|
&mut context,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
4096,
|
||||||
|
128,
|
||||||
|
64,
|
||||||
|
&mut needs_index_reset,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(unsafe { ZSTD_rust_resetCCtxWorkspace(&state) }, 0);
|
||||||
|
assert_eq!(needs_index_reset, 0);
|
||||||
|
assert_eq!(
|
||||||
|
context.events,
|
||||||
|
[
|
||||||
|
ResetCCtxWorkspaceTestEvent::BumpOversizedDuration,
|
||||||
|
ResetCCtxWorkspaceTestEvent::ClearWorkspace,
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cctx_workspace_preserves_resize_reservation_and_publication_order() {
|
||||||
|
let mut context = ResetCCtxWorkspaceTestContext::default();
|
||||||
|
let mut needs_index_reset = 0;
|
||||||
|
let state = reset_cctx_workspace_test_state(
|
||||||
|
&mut context,
|
||||||
|
0,
|
||||||
|
1,
|
||||||
|
0,
|
||||||
|
4096,
|
||||||
|
128,
|
||||||
|
64,
|
||||||
|
&mut needs_index_reset,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(unsafe { ZSTD_rust_resetCCtxWorkspace(&state) }, 0);
|
||||||
|
assert_eq!(needs_index_reset, RESET_CCTX_INDEX_RESET);
|
||||||
|
assert_eq!(
|
||||||
|
context.events,
|
||||||
|
[
|
||||||
|
ResetCCtxWorkspaceTestEvent::BumpOversizedDuration,
|
||||||
|
ResetCCtxWorkspaceTestEvent::FreeWorkspace,
|
||||||
|
ResetCCtxWorkspaceTestEvent::CreateWorkspace(4096),
|
||||||
|
ResetCCtxWorkspaceTestEvent::ReserveObject(128),
|
||||||
|
ResetCCtxWorkspaceTestEvent::Pointer(RESET_CCTX_POINTER_PREV_CBLOCK),
|
||||||
|
ResetCCtxWorkspaceTestEvent::ReserveObject(128),
|
||||||
|
ResetCCtxWorkspaceTestEvent::Pointer(RESET_CCTX_POINTER_NEXT_CBLOCK),
|
||||||
|
ResetCCtxWorkspaceTestEvent::ReserveObject(64),
|
||||||
|
ResetCCtxWorkspaceTestEvent::Pointer(RESET_CCTX_POINTER_TMP_WORKSPACE),
|
||||||
|
ResetCCtxWorkspaceTestEvent::Size(RESET_CCTX_SIZE_TMP_WORKSPACE, 64),
|
||||||
|
ResetCCtxWorkspaceTestEvent::ClearWorkspace,
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cctx_workspace_rejects_static_resize_before_mutating_workspace() {
|
||||||
|
let mut context = ResetCCtxWorkspaceTestContext::default();
|
||||||
|
let mut needs_index_reset = 0;
|
||||||
|
let state = reset_cctx_workspace_test_state(
|
||||||
|
&mut context,
|
||||||
|
1,
|
||||||
|
1,
|
||||||
|
0,
|
||||||
|
4096,
|
||||||
|
128,
|
||||||
|
64,
|
||||||
|
&mut needs_index_reset,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
unsafe { ZSTD_rust_resetCCtxWorkspace(&state) },
|
||||||
|
ERROR(ZstdErrorCode::MemoryAllocation)
|
||||||
|
);
|
||||||
|
assert_eq!(needs_index_reset, 0);
|
||||||
|
assert!(context.events.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn estimate_cctx_workspace_size_keeps_static_and_buffer_components_separate() {
|
fn estimate_cctx_workspace_size_keeps_static_and_buffer_components_separate() {
|
||||||
let sizing = cctx_workspace_test_sizing();
|
let sizing = cctx_workspace_test_sizing();
|
||||||
|
|||||||
Reference in New Issue
Block a user