feat(compress): move CCtx workspace policy into Rust
Move the CCtx reset workspace resize decision, index-reset mutation, reservation order, private-pointer publication order, and clear boundary into Rust. Keep the workspace allocator and private ZSTD_CCtx layout operations in C behind callbacks, with matching C/Rust layout assertions. Add focused tests for the no-resize, resize, and static-resize error paths so the callback order and mutation boundaries remain explicit. Test Plan: - `ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml` - `ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings` - `ulimit -v 41943040 && make -j1` - `ulimit -v 41943040 && make -j1 -C tests test-zstream ZSTREAM_TESTTIME=-T2s` - `ulimit -v 41943040 && make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests` - `ulimit -v 41943040 && cargo fmt --manifest-path rust/Cargo.toml -- --check` - `git diff --check`
This commit is contained in:
@@ -6574,6 +6574,9 @@ const RESET_CCTX_POINTER_LL_CODE: c_int = 7;
|
||||
const RESET_CCTX_POINTER_ML_CODE: c_int = 8;
|
||||
const RESET_CCTX_POINTER_OF_CODE: c_int = 9;
|
||||
const RESET_CCTX_POINTER_LDM_BUCKETS: c_int = 10;
|
||||
const RESET_CCTX_POINTER_PREV_CBLOCK: c_int = 11;
|
||||
const RESET_CCTX_POINTER_NEXT_CBLOCK: c_int = 12;
|
||||
const RESET_CCTX_POINTER_TMP_WORKSPACE: c_int = 13;
|
||||
|
||||
const RESET_CCTX_SIZE_MAX_NB_SEQ: c_int = 0;
|
||||
const RESET_CCTX_SIZE_MAX_NB_LIT: c_int = 1;
|
||||
@@ -6581,9 +6584,11 @@ const RESET_CCTX_SIZE_MAX_NB_LDM_SEQ: c_int = 2;
|
||||
const RESET_CCTX_SIZE_EXTERNAL_SEQ_CAPACITY: c_int = 3;
|
||||
const RESET_CCTX_SIZE_INPUT_BUFFER: c_int = 4;
|
||||
const RESET_CCTX_SIZE_OUTPUT_BUFFER: c_int = 5;
|
||||
const RESET_CCTX_SIZE_TMP_WORKSPACE: c_int = 6;
|
||||
|
||||
const RESET_CCTX_INT_BUFFERED_POLICY: c_int = 0;
|
||||
const RESET_CCTX_INT_INITIALIZED: c_int = 1;
|
||||
const RESET_CCTX_INDEX_RESET: c_int = 1;
|
||||
|
||||
/// C-owned workspace/layout projection for the post-match-state reset tail.
|
||||
///
|
||||
@@ -6713,6 +6718,88 @@ const _: () = {
|
||||
);
|
||||
};
|
||||
|
||||
type ResetCCtxWorkspaceCreate = unsafe extern "C" fn(*mut c_void, usize) -> usize;
|
||||
type ResetCCtxWorkspaceReserveObject = unsafe extern "C" fn(*mut c_void, usize) -> *mut c_void;
|
||||
|
||||
/// C-owned workspace operations used by the CCtx reset policy.
|
||||
///
|
||||
/// Rust owns the resize decision, error ordering, object-reservation order,
|
||||
/// and workspace-clear boundary. C retains the workspace representation,
|
||||
/// allocator, and private CCtx publication callbacks.
|
||||
#[repr(C)]
|
||||
pub struct ZSTD_rust_resetCCtxWorkspaceState {
|
||||
callbackContext: *mut c_void,
|
||||
isStatic: c_int,
|
||||
workspaceTooSmall: c_int,
|
||||
workspaceWasteful: c_int,
|
||||
neededSpace: usize,
|
||||
compressedBlockStateSize: usize,
|
||||
tmpWorkspaceSize: usize,
|
||||
needsIndexReset: *mut c_int,
|
||||
bumpOversizedDuration: Option<ResetCCtxStorageCallback>,
|
||||
freeWorkspace: Option<ResetCCtxStorageCallback>,
|
||||
createWorkspace: Option<ResetCCtxWorkspaceCreate>,
|
||||
reserveObject: Option<ResetCCtxWorkspaceReserveObject>,
|
||||
setPointer: Option<ResetCCtxStorageSetPointer>,
|
||||
setSize: Option<ResetCCtxStorageSetSize>,
|
||||
clearWorkspace: Option<ResetCCtxStorageCallback>,
|
||||
}
|
||||
|
||||
const _: () = {
|
||||
assert!(offset_of!(ZSTD_rust_resetCCtxWorkspaceState, callbackContext) == 0);
|
||||
assert!(offset_of!(ZSTD_rust_resetCCtxWorkspaceState, isStatic) == size_of::<usize>());
|
||||
assert!(
|
||||
offset_of!(ZSTD_rust_resetCCtxWorkspaceState, workspaceTooSmall)
|
||||
== size_of::<usize>() + size_of::<c_int>()
|
||||
);
|
||||
assert!(
|
||||
offset_of!(ZSTD_rust_resetCCtxWorkspaceState, workspaceWasteful)
|
||||
== size_of::<usize>() + 2 * size_of::<c_int>()
|
||||
);
|
||||
assert!(
|
||||
offset_of!(ZSTD_rust_resetCCtxWorkspaceState, neededSpace)
|
||||
> offset_of!(ZSTD_rust_resetCCtxWorkspaceState, workspaceWasteful)
|
||||
);
|
||||
assert!(
|
||||
offset_of!(ZSTD_rust_resetCCtxWorkspaceState, compressedBlockStateSize)
|
||||
== offset_of!(ZSTD_rust_resetCCtxWorkspaceState, neededSpace) + size_of::<usize>()
|
||||
);
|
||||
assert!(
|
||||
offset_of!(ZSTD_rust_resetCCtxWorkspaceState, tmpWorkspaceSize)
|
||||
== offset_of!(ZSTD_rust_resetCCtxWorkspaceState, compressedBlockStateSize)
|
||||
+ size_of::<usize>()
|
||||
);
|
||||
assert!(
|
||||
offset_of!(ZSTD_rust_resetCCtxWorkspaceState, needsIndexReset)
|
||||
== offset_of!(ZSTD_rust_resetCCtxWorkspaceState, tmpWorkspaceSize) + size_of::<usize>()
|
||||
);
|
||||
assert!(
|
||||
offset_of!(ZSTD_rust_resetCCtxWorkspaceState, freeWorkspace)
|
||||
== offset_of!(ZSTD_rust_resetCCtxWorkspaceState, bumpOversizedDuration)
|
||||
+ size_of::<usize>()
|
||||
);
|
||||
assert!(
|
||||
offset_of!(ZSTD_rust_resetCCtxWorkspaceState, createWorkspace)
|
||||
== offset_of!(ZSTD_rust_resetCCtxWorkspaceState, freeWorkspace) + size_of::<usize>()
|
||||
);
|
||||
assert!(
|
||||
offset_of!(ZSTD_rust_resetCCtxWorkspaceState, reserveObject)
|
||||
== offset_of!(ZSTD_rust_resetCCtxWorkspaceState, createWorkspace) + size_of::<usize>()
|
||||
);
|
||||
assert!(
|
||||
offset_of!(ZSTD_rust_resetCCtxWorkspaceState, setPointer)
|
||||
== offset_of!(ZSTD_rust_resetCCtxWorkspaceState, reserveObject) + size_of::<usize>()
|
||||
);
|
||||
assert!(
|
||||
offset_of!(ZSTD_rust_resetCCtxWorkspaceState, setSize)
|
||||
== offset_of!(ZSTD_rust_resetCCtxWorkspaceState, setPointer) + size_of::<usize>()
|
||||
);
|
||||
assert!(
|
||||
size_of::<ZSTD_rust_resetCCtxWorkspaceState>()
|
||||
== offset_of!(ZSTD_rust_resetCCtxWorkspaceState, clearWorkspace) + size_of::<usize>()
|
||||
);
|
||||
};
|
||||
|
||||
/// Reserve and publish the private CCtx storage that follows match-state reset.
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_rust_resetCCtxStorage(
|
||||
@@ -6909,6 +6996,92 @@ pub unsafe extern "C" fn ZSTD_rust_resetCCtxStorage(
|
||||
0
|
||||
}
|
||||
|
||||
/// Own the CCtx workspace resize, object reservation, and clear order.
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_rust_resetCCtxWorkspace(
|
||||
state: *const ZSTD_rust_resetCCtxWorkspaceState,
|
||||
) -> usize {
|
||||
if state.is_null() {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
}
|
||||
let state = unsafe { &*state };
|
||||
if state.callbackContext.is_null() || state.needsIndexReset.is_null() {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
}
|
||||
let Some(bump_oversized_duration) = state.bumpOversizedDuration else {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
};
|
||||
let Some(free_workspace) = state.freeWorkspace else {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
};
|
||||
let Some(create_workspace) = state.createWorkspace else {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
};
|
||||
let Some(reserve_object) = state.reserveObject else {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
};
|
||||
let Some(set_pointer) = state.setPointer else {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
};
|
||||
let Some(set_size) = state.setSize else {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
};
|
||||
let Some(clear_workspace) = state.clearWorkspace else {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
};
|
||||
|
||||
unsafe {
|
||||
if state.isStatic == 0 {
|
||||
bump_oversized_duration(state.callbackContext);
|
||||
}
|
||||
let resize_workspace = state.workspaceTooSmall != 0 || state.workspaceWasteful != 0;
|
||||
if resize_workspace {
|
||||
if state.isStatic != 0 {
|
||||
return ERROR(ZstdErrorCode::MemoryAllocation);
|
||||
}
|
||||
*state.needsIndexReset = RESET_CCTX_INDEX_RESET;
|
||||
free_workspace(state.callbackContext);
|
||||
let result = create_workspace(state.callbackContext, state.neededSpace);
|
||||
if ERR_isError(result) {
|
||||
return result;
|
||||
}
|
||||
|
||||
let previous = reserve_object(state.callbackContext, state.compressedBlockStateSize);
|
||||
if previous.is_null() {
|
||||
return ERROR(ZstdErrorCode::MemoryAllocation);
|
||||
}
|
||||
set_pointer(
|
||||
state.callbackContext,
|
||||
RESET_CCTX_POINTER_PREV_CBLOCK,
|
||||
previous,
|
||||
);
|
||||
|
||||
let next = reserve_object(state.callbackContext, state.compressedBlockStateSize);
|
||||
if next.is_null() {
|
||||
return ERROR(ZstdErrorCode::MemoryAllocation);
|
||||
}
|
||||
set_pointer(state.callbackContext, RESET_CCTX_POINTER_NEXT_CBLOCK, next);
|
||||
|
||||
let tmp_workspace = reserve_object(state.callbackContext, state.tmpWorkspaceSize);
|
||||
if tmp_workspace.is_null() {
|
||||
return ERROR(ZstdErrorCode::MemoryAllocation);
|
||||
}
|
||||
set_pointer(
|
||||
state.callbackContext,
|
||||
RESET_CCTX_POINTER_TMP_WORKSPACE,
|
||||
tmp_workspace,
|
||||
);
|
||||
set_size(
|
||||
state.callbackContext,
|
||||
RESET_CCTX_SIZE_TMP_WORKSPACE,
|
||||
state.tmpWorkspaceSize,
|
||||
);
|
||||
}
|
||||
clear_workspace(state.callbackContext);
|
||||
}
|
||||
0
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn max_estimate_cctx_size(
|
||||
estimate0: usize,
|
||||
@@ -16276,6 +16449,210 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq)]
|
||||
enum ResetCCtxWorkspaceTestEvent {
|
||||
BumpOversizedDuration,
|
||||
FreeWorkspace,
|
||||
CreateWorkspace(usize),
|
||||
ReserveObject(usize),
|
||||
Pointer(c_int),
|
||||
Size(c_int, usize),
|
||||
ClearWorkspace,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ResetCCtxWorkspaceTestContext {
|
||||
events: Vec<ResetCCtxWorkspaceTestEvent>,
|
||||
reserve_count: usize,
|
||||
create_result: usize,
|
||||
}
|
||||
|
||||
unsafe fn reset_cctx_workspace_test_context(
|
||||
context: *mut c_void,
|
||||
) -> &'static mut ResetCCtxWorkspaceTestContext {
|
||||
unsafe { &mut *context.cast::<ResetCCtxWorkspaceTestContext>() }
|
||||
}
|
||||
|
||||
unsafe extern "C" fn reset_cctx_workspace_test_bump(context: *mut c_void) {
|
||||
let context = unsafe { reset_cctx_workspace_test_context(context) };
|
||||
context
|
||||
.events
|
||||
.push(ResetCCtxWorkspaceTestEvent::BumpOversizedDuration);
|
||||
}
|
||||
|
||||
unsafe extern "C" fn reset_cctx_workspace_test_free(context: *mut c_void) {
|
||||
let context = unsafe { reset_cctx_workspace_test_context(context) };
|
||||
context
|
||||
.events
|
||||
.push(ResetCCtxWorkspaceTestEvent::FreeWorkspace);
|
||||
}
|
||||
|
||||
unsafe extern "C" fn reset_cctx_workspace_test_create(
|
||||
context: *mut c_void,
|
||||
needed_space: usize,
|
||||
) -> usize {
|
||||
let context = unsafe { reset_cctx_workspace_test_context(context) };
|
||||
context
|
||||
.events
|
||||
.push(ResetCCtxWorkspaceTestEvent::CreateWorkspace(needed_space));
|
||||
context.create_result
|
||||
}
|
||||
|
||||
unsafe extern "C" fn reset_cctx_workspace_test_reserve_object(
|
||||
context: *mut c_void,
|
||||
size: usize,
|
||||
) -> *mut c_void {
|
||||
let context = unsafe { reset_cctx_workspace_test_context(context) };
|
||||
context
|
||||
.events
|
||||
.push(ResetCCtxWorkspaceTestEvent::ReserveObject(size));
|
||||
let pointer = (0x1000usize + context.reserve_count * 0x1000) as *mut c_void;
|
||||
context.reserve_count += 1;
|
||||
pointer
|
||||
}
|
||||
|
||||
unsafe extern "C" fn reset_cctx_workspace_test_set_pointer(
|
||||
context: *mut c_void,
|
||||
kind: c_int,
|
||||
_pointer: *mut c_void,
|
||||
) {
|
||||
let context = unsafe { reset_cctx_workspace_test_context(context) };
|
||||
context
|
||||
.events
|
||||
.push(ResetCCtxWorkspaceTestEvent::Pointer(kind));
|
||||
}
|
||||
|
||||
unsafe extern "C" fn reset_cctx_workspace_test_set_size(
|
||||
context: *mut c_void,
|
||||
kind: c_int,
|
||||
value: usize,
|
||||
) {
|
||||
let context = unsafe { reset_cctx_workspace_test_context(context) };
|
||||
context
|
||||
.events
|
||||
.push(ResetCCtxWorkspaceTestEvent::Size(kind, value));
|
||||
}
|
||||
|
||||
unsafe extern "C" fn reset_cctx_workspace_test_clear(context: *mut c_void) {
|
||||
let context = unsafe { reset_cctx_workspace_test_context(context) };
|
||||
context
|
||||
.events
|
||||
.push(ResetCCtxWorkspaceTestEvent::ClearWorkspace);
|
||||
}
|
||||
|
||||
fn reset_cctx_workspace_test_state(
|
||||
context: &mut ResetCCtxWorkspaceTestContext,
|
||||
is_static: c_int,
|
||||
workspace_too_small: c_int,
|
||||
workspace_wasteful: c_int,
|
||||
needed_space: usize,
|
||||
compressed_block_state_size: usize,
|
||||
tmp_workspace_size: usize,
|
||||
needs_index_reset: &mut c_int,
|
||||
) -> ZSTD_rust_resetCCtxWorkspaceState {
|
||||
ZSTD_rust_resetCCtxWorkspaceState {
|
||||
callbackContext: (context as *mut ResetCCtxWorkspaceTestContext).cast(),
|
||||
isStatic: is_static,
|
||||
workspaceTooSmall: workspace_too_small,
|
||||
workspaceWasteful: workspace_wasteful,
|
||||
neededSpace: needed_space,
|
||||
compressedBlockStateSize: compressed_block_state_size,
|
||||
tmpWorkspaceSize: tmp_workspace_size,
|
||||
needsIndexReset: needs_index_reset,
|
||||
bumpOversizedDuration: Some(reset_cctx_workspace_test_bump),
|
||||
freeWorkspace: Some(reset_cctx_workspace_test_free),
|
||||
createWorkspace: Some(reset_cctx_workspace_test_create),
|
||||
reserveObject: Some(reset_cctx_workspace_test_reserve_object),
|
||||
setPointer: Some(reset_cctx_workspace_test_set_pointer),
|
||||
setSize: Some(reset_cctx_workspace_test_set_size),
|
||||
clearWorkspace: Some(reset_cctx_workspace_test_clear),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cctx_workspace_preserves_existing_workspace_and_clear_order() {
|
||||
let mut context = ResetCCtxWorkspaceTestContext::default();
|
||||
let mut needs_index_reset = 0;
|
||||
let state = reset_cctx_workspace_test_state(
|
||||
&mut context,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
4096,
|
||||
128,
|
||||
64,
|
||||
&mut needs_index_reset,
|
||||
);
|
||||
|
||||
assert_eq!(unsafe { ZSTD_rust_resetCCtxWorkspace(&state) }, 0);
|
||||
assert_eq!(needs_index_reset, 0);
|
||||
assert_eq!(
|
||||
context.events,
|
||||
[
|
||||
ResetCCtxWorkspaceTestEvent::BumpOversizedDuration,
|
||||
ResetCCtxWorkspaceTestEvent::ClearWorkspace,
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cctx_workspace_preserves_resize_reservation_and_publication_order() {
|
||||
let mut context = ResetCCtxWorkspaceTestContext::default();
|
||||
let mut needs_index_reset = 0;
|
||||
let state = reset_cctx_workspace_test_state(
|
||||
&mut context,
|
||||
0,
|
||||
1,
|
||||
0,
|
||||
4096,
|
||||
128,
|
||||
64,
|
||||
&mut needs_index_reset,
|
||||
);
|
||||
|
||||
assert_eq!(unsafe { ZSTD_rust_resetCCtxWorkspace(&state) }, 0);
|
||||
assert_eq!(needs_index_reset, RESET_CCTX_INDEX_RESET);
|
||||
assert_eq!(
|
||||
context.events,
|
||||
[
|
||||
ResetCCtxWorkspaceTestEvent::BumpOversizedDuration,
|
||||
ResetCCtxWorkspaceTestEvent::FreeWorkspace,
|
||||
ResetCCtxWorkspaceTestEvent::CreateWorkspace(4096),
|
||||
ResetCCtxWorkspaceTestEvent::ReserveObject(128),
|
||||
ResetCCtxWorkspaceTestEvent::Pointer(RESET_CCTX_POINTER_PREV_CBLOCK),
|
||||
ResetCCtxWorkspaceTestEvent::ReserveObject(128),
|
||||
ResetCCtxWorkspaceTestEvent::Pointer(RESET_CCTX_POINTER_NEXT_CBLOCK),
|
||||
ResetCCtxWorkspaceTestEvent::ReserveObject(64),
|
||||
ResetCCtxWorkspaceTestEvent::Pointer(RESET_CCTX_POINTER_TMP_WORKSPACE),
|
||||
ResetCCtxWorkspaceTestEvent::Size(RESET_CCTX_SIZE_TMP_WORKSPACE, 64),
|
||||
ResetCCtxWorkspaceTestEvent::ClearWorkspace,
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cctx_workspace_rejects_static_resize_before_mutating_workspace() {
|
||||
let mut context = ResetCCtxWorkspaceTestContext::default();
|
||||
let mut needs_index_reset = 0;
|
||||
let state = reset_cctx_workspace_test_state(
|
||||
&mut context,
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
4096,
|
||||
128,
|
||||
64,
|
||||
&mut needs_index_reset,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
unsafe { ZSTD_rust_resetCCtxWorkspace(&state) },
|
||||
ERROR(ZstdErrorCode::MemoryAllocation)
|
||||
);
|
||||
assert_eq!(needs_index_reset, 0);
|
||||
assert!(context.events.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn estimate_cctx_workspace_size_keeps_static_and_buffer_components_separate() {
|
||||
let sizing = cctx_workspace_test_sizing();
|
||||
|
||||
Reference in New Issue
Block a user