feat(compress): move CCtx block-state copy into Rust
Pass the destination compressed-block-state pointer slot and source state through the copyCCtx bridge, then perform the final copy in Rust after reset has established the destination. Remove the redundant C memcpy callback while preserving private callback ordering and early-error behavior. Add focused tests for copied repcodes and entropy repeat state. Test Plan: - ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml - ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040 && make -j1 - ulimit -v 41943040 && make -j1 -C tests test-zstream ZSTREAM_TESTTIME=-T2s - ulimit -v 41943040 && make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests - ulimit -v 41943040 && cargo fmt --manifest-path rust/Cargo.toml -- --check - git diff --check
This commit is contained in:
@@ -481,7 +481,8 @@ typedef struct {
|
|||||||
ZSTD_rust_copyCCtxMarkTables_f markTablesClean;
|
ZSTD_rust_copyCCtxMarkTables_f markTablesClean;
|
||||||
ZSTD_rust_copyCCtxCopyState_f copyMatchState;
|
ZSTD_rust_copyCCtxCopyState_f copyMatchState;
|
||||||
ZSTD_rust_copyCCtxCopyState_f copyDictState;
|
ZSTD_rust_copyCCtxCopyState_f copyDictState;
|
||||||
ZSTD_rust_copyCCtxCopyState_f copyBlockState;
|
ZSTD_compressedBlockState_t** destinationBlockState;
|
||||||
|
const ZSTD_compressedBlockState_t* sourceBlockState;
|
||||||
int zbuff;
|
int zbuff;
|
||||||
} ZSTD_rust_copyCCtxInternalState;
|
} ZSTD_rust_copyCCtxInternalState;
|
||||||
size_t ZSTD_rust_copyCCtxInternal(
|
size_t ZSTD_rust_copyCCtxInternal(
|
||||||
@@ -522,7 +523,7 @@ typedef char ZSTD_rust_copy_cctx_internal_state_layout[
|
|||||||
== 3 * sizeof(void*) + sizeof(U64)
|
== 3 * sizeof(void*) + sizeof(U64)
|
||||||
&& offsetof(ZSTD_rust_copyCCtxInternalState, zbuff)
|
&& offsetof(ZSTD_rust_copyCCtxInternalState, zbuff)
|
||||||
== 3 * sizeof(void*) + sizeof(U64)
|
== 3 * sizeof(void*) + sizeof(U64)
|
||||||
+ 9 * sizeof(void*)
|
+ 10 * sizeof(void*)
|
||||||
&& sizeof(ZSTD_rust_copyCCtxInternalState)
|
&& sizeof(ZSTD_rust_copyCCtxInternalState)
|
||||||
== ((offsetof(ZSTD_rust_copyCCtxInternalState, zbuff)
|
== ((offsetof(ZSTD_rust_copyCCtxInternalState, zbuff)
|
||||||
+ sizeof(int) + sizeof(void*) - 1) / sizeof(void*))
|
+ sizeof(int) + sizeof(void*) - 1) / sizeof(void*))
|
||||||
@@ -4854,16 +4855,6 @@ static void ZSTD_rust_copyCCtx_copy_dict_state(
|
|||||||
dst->dictContentSize = src->dictContentSize;
|
dst->dictContentSize = src->dictContentSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void ZSTD_rust_copyCCtx_copy_block_state(
|
|
||||||
void* context, const void* srcCCtx)
|
|
||||||
{
|
|
||||||
ZSTD_CCtx* const dst = (ZSTD_CCtx*)context;
|
|
||||||
const ZSTD_CCtx* const src = (const ZSTD_CCtx*)srcCCtx;
|
|
||||||
ZSTD_memcpy(dst->blockState.prevCBlock,
|
|
||||||
src->blockState.prevCBlock,
|
|
||||||
sizeof(*src->blockState.prevCBlock));
|
|
||||||
}
|
|
||||||
|
|
||||||
static size_t ZSTD_rust_copyCCtx_internal_callback(
|
static size_t ZSTD_rust_copyCCtx_internal_callback(
|
||||||
void* context, const void* srcCCtx,
|
void* context, const void* srcCCtx,
|
||||||
const ZSTD_frameParameters* fParams,
|
const ZSTD_frameParameters* fParams,
|
||||||
@@ -4882,7 +4873,8 @@ static size_t ZSTD_rust_copyCCtx_internal_callback(
|
|||||||
state.markTablesClean = ZSTD_rust_copyCCtx_mark_tables_clean;
|
state.markTablesClean = ZSTD_rust_copyCCtx_mark_tables_clean;
|
||||||
state.copyMatchState = ZSTD_rust_copyCCtx_copy_match_state;
|
state.copyMatchState = ZSTD_rust_copyCCtx_copy_match_state;
|
||||||
state.copyDictState = ZSTD_rust_copyCCtx_copy_dict_state;
|
state.copyDictState = ZSTD_rust_copyCCtx_copy_dict_state;
|
||||||
state.copyBlockState = ZSTD_rust_copyCCtx_copy_block_state;
|
state.destinationBlockState = &((ZSTD_CCtx*)context)->blockState.prevCBlock;
|
||||||
|
state.sourceBlockState = ((const ZSTD_CCtx*)srcCCtx)->blockState.prevCBlock;
|
||||||
state.zbuff = zbuff;
|
state.zbuff = zbuff;
|
||||||
return ZSTD_rust_copyCCtxInternal(&state);
|
return ZSTD_rust_copyCCtxInternal(&state);
|
||||||
}
|
}
|
||||||
|
|||||||
+72
-16
@@ -1703,7 +1703,8 @@ pub struct ZSTD_rust_copyCCtxInternalState {
|
|||||||
mark_tables_clean: Option<CopyCCtxMarkTablesFn>,
|
mark_tables_clean: Option<CopyCCtxMarkTablesFn>,
|
||||||
copy_match_state: Option<CopyCCtxCopyStateFn>,
|
copy_match_state: Option<CopyCCtxCopyStateFn>,
|
||||||
copy_dict_state: Option<CopyCCtxCopyStateFn>,
|
copy_dict_state: Option<CopyCCtxCopyStateFn>,
|
||||||
copy_block_state: Option<CopyCCtxCopyStateFn>,
|
destination_block_state: *mut *mut ZSTD_compressedBlockState_t,
|
||||||
|
source_block_state: *const ZSTD_compressedBlockState_t,
|
||||||
zbuff: c_int,
|
zbuff: c_int,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1725,7 +1726,7 @@ const _: () = {
|
|||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
offset_of!(ZSTD_rust_copyCCtxInternalState, zbuff)
|
offset_of!(ZSTD_rust_copyCCtxInternalState, zbuff)
|
||||||
== 3 * size_of::<usize>() + size_of::<u64>() + 9 * size_of::<usize>()
|
== 3 * size_of::<usize>() + size_of::<u64>() + 10 * size_of::<usize>()
|
||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
size_of::<ZSTD_rust_copyCCtxInternalState>()
|
size_of::<ZSTD_rust_copyCCtxInternalState>()
|
||||||
@@ -1735,7 +1736,8 @@ const _: () = {
|
|||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Run the private context-copy operation through C-owned layout callbacks.
|
/// Run the private context-copy operation through C-owned layout callbacks
|
||||||
|
/// and copy the compressed-block state directly in Rust.
|
||||||
#[no_mangle]
|
#[no_mangle]
|
||||||
pub unsafe extern "C" fn ZSTD_rust_copyCCtxInternal(
|
pub unsafe extern "C" fn ZSTD_rust_copyCCtxInternal(
|
||||||
state: *const ZSTD_rust_copyCCtxInternalState,
|
state: *const ZSTD_rust_copyCCtxInternalState,
|
||||||
@@ -1753,7 +1755,6 @@ pub unsafe extern "C" fn ZSTD_rust_copyCCtxInternal(
|
|||||||
Some(mark_tables_clean),
|
Some(mark_tables_clean),
|
||||||
Some(copy_match_state),
|
Some(copy_match_state),
|
||||||
Some(copy_dict_state),
|
Some(copy_dict_state),
|
||||||
Some(copy_block_state),
|
|
||||||
) = (
|
) = (
|
||||||
state.check_stage,
|
state.check_stage,
|
||||||
state.copy_custom_mem,
|
state.copy_custom_mem,
|
||||||
@@ -1763,12 +1764,16 @@ pub unsafe extern "C" fn ZSTD_rust_copyCCtxInternal(
|
|||||||
state.mark_tables_clean,
|
state.mark_tables_clean,
|
||||||
state.copy_match_state,
|
state.copy_match_state,
|
||||||
state.copy_dict_state,
|
state.copy_dict_state,
|
||||||
state.copy_block_state,
|
|
||||||
)
|
)
|
||||||
else {
|
else {
|
||||||
return ERROR(ZstdErrorCode::Generic);
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
};
|
};
|
||||||
if state.callback_context.is_null() || state.src_cctx.is_null() || state.f_params.is_null() {
|
if state.callback_context.is_null()
|
||||||
|
|| state.src_cctx.is_null()
|
||||||
|
|| state.f_params.is_null()
|
||||||
|
|| state.destination_block_state.is_null()
|
||||||
|
|| state.source_block_state.is_null()
|
||||||
|
{
|
||||||
return ERROR(ZstdErrorCode::Generic);
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1794,7 +1799,11 @@ pub unsafe extern "C" fn ZSTD_rust_copyCCtxInternal(
|
|||||||
mark_tables_clean(state.callback_context);
|
mark_tables_clean(state.callback_context);
|
||||||
copy_match_state(state.callback_context, state.src_cctx);
|
copy_match_state(state.callback_context, state.src_cctx);
|
||||||
copy_dict_state(state.callback_context, state.src_cctx);
|
copy_dict_state(state.callback_context, state.src_cctx);
|
||||||
copy_block_state(state.callback_context, state.src_cctx);
|
let destination_block_state = *state.destination_block_state;
|
||||||
|
if destination_block_state.is_null() {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
}
|
||||||
|
ptr::copy_nonoverlapping(state.source_block_state, destination_block_state, 1);
|
||||||
}
|
}
|
||||||
0
|
0
|
||||||
}
|
}
|
||||||
@@ -13863,7 +13872,6 @@ mod tests {
|
|||||||
1 => "custom",
|
1 => "custom",
|
||||||
6 => "match",
|
6 => "match",
|
||||||
7 => "dict",
|
7 => "dict",
|
||||||
8 => "block",
|
|
||||||
_ => panic!("unexpected copy-state callback order"),
|
_ => panic!("unexpected copy-state callback order"),
|
||||||
};
|
};
|
||||||
context.events.push(event);
|
context.events.push(event);
|
||||||
@@ -13908,6 +13916,8 @@ mod tests {
|
|||||||
context: &mut CopyCCtxInternalTestContext,
|
context: &mut CopyCCtxInternalTestContext,
|
||||||
src_cctx: *const c_void,
|
src_cctx: *const c_void,
|
||||||
f_params: &ZSTD_frameParameters,
|
f_params: &ZSTD_frameParameters,
|
||||||
|
destination_block_state: *mut *mut ZSTD_compressedBlockState_t,
|
||||||
|
source_block_state: *const ZSTD_compressedBlockState_t,
|
||||||
) -> ZSTD_rust_copyCCtxInternalState {
|
) -> ZSTD_rust_copyCCtxInternalState {
|
||||||
ZSTD_rust_copyCCtxInternalState {
|
ZSTD_rust_copyCCtxInternalState {
|
||||||
callback_context: (context as *mut CopyCCtxInternalTestContext).cast(),
|
callback_context: (context as *mut CopyCCtxInternalTestContext).cast(),
|
||||||
@@ -13922,7 +13932,8 @@ mod tests {
|
|||||||
mark_tables_clean: Some(copy_cctx_internal_test_mark_tables_clean),
|
mark_tables_clean: Some(copy_cctx_internal_test_mark_tables_clean),
|
||||||
copy_match_state: Some(copy_cctx_internal_test_copy_state),
|
copy_match_state: Some(copy_cctx_internal_test_copy_state),
|
||||||
copy_dict_state: Some(copy_cctx_internal_test_copy_state),
|
copy_dict_state: Some(copy_cctx_internal_test_copy_state),
|
||||||
copy_block_state: Some(copy_cctx_internal_test_copy_state),
|
destination_block_state,
|
||||||
|
source_block_state,
|
||||||
zbuff: 7,
|
zbuff: 7,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -13935,19 +13946,42 @@ mod tests {
|
|||||||
checksumFlag: 1,
|
checksumFlag: 1,
|
||||||
noDictIDFlag: 1,
|
noDictIDFlag: 1,
|
||||||
};
|
};
|
||||||
let state =
|
let mut source_block_state =
|
||||||
copy_cctx_internal_test_state(&mut context, 0x5000usize as *const c_void, &f_params);
|
unsafe { MaybeUninit::<ZSTD_compressedBlockState_t>::zeroed().assume_init() };
|
||||||
|
source_block_state.rep = [15, 16, 17];
|
||||||
|
source_block_state.entropy.huf.repeatMode = 2;
|
||||||
|
source_block_state.entropy.fse.offcode_repeatMode = FSE_REPEAT_VALID;
|
||||||
|
let mut destination_block_state =
|
||||||
|
unsafe { MaybeUninit::<ZSTD_compressedBlockState_t>::zeroed().assume_init() };
|
||||||
|
let mut destination_block_state_slot =
|
||||||
|
&mut destination_block_state as *mut ZSTD_compressedBlockState_t;
|
||||||
|
let state = copy_cctx_internal_test_state(
|
||||||
|
&mut context,
|
||||||
|
0x5000usize as *const c_void,
|
||||||
|
&f_params,
|
||||||
|
&mut destination_block_state_slot,
|
||||||
|
&source_block_state,
|
||||||
|
);
|
||||||
|
|
||||||
let result = unsafe { ZSTD_rust_copyCCtxInternal(&state) };
|
let result = unsafe { ZSTD_rust_copyCCtxInternal(&state) };
|
||||||
|
|
||||||
assert_eq!(result, 0);
|
assert_eq!(result, 0);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
context.events,
|
context.events,
|
||||||
["check", "custom", "reset", "dirty", "tables", "clean", "match", "dict", "block"]
|
["check", "custom", "reset", "dirty", "tables", "clean", "match", "dict"]
|
||||||
);
|
);
|
||||||
assert_eq!(context.frame_params, f_params);
|
assert_eq!(context.frame_params, f_params);
|
||||||
assert_eq!(context.pledged_src_size, 123);
|
assert_eq!(context.pledged_src_size, 123);
|
||||||
assert_eq!(context.zbuff, 7);
|
assert_eq!(context.zbuff, 7);
|
||||||
|
assert_eq!(destination_block_state.rep, source_block_state.rep);
|
||||||
|
assert_eq!(
|
||||||
|
destination_block_state.entropy.huf.repeatMode,
|
||||||
|
source_block_state.entropy.huf.repeatMode
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
destination_block_state.entropy.fse.offcode_repeatMode,
|
||||||
|
source_block_state.entropy.fse.offcode_repeatMode
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -13957,8 +13991,19 @@ mod tests {
|
|||||||
..CopyCCtxInternalTestContext::default()
|
..CopyCCtxInternalTestContext::default()
|
||||||
};
|
};
|
||||||
let f_params = ZSTD_frameParameters::default();
|
let f_params = ZSTD_frameParameters::default();
|
||||||
let state =
|
let mut destination_block_state =
|
||||||
copy_cctx_internal_test_state(&mut context, 0x5000usize as *const c_void, &f_params);
|
unsafe { MaybeUninit::<ZSTD_compressedBlockState_t>::zeroed().assume_init() };
|
||||||
|
let mut destination_block_state_slot =
|
||||||
|
&mut destination_block_state as *mut ZSTD_compressedBlockState_t;
|
||||||
|
let source_block_state =
|
||||||
|
unsafe { MaybeUninit::<ZSTD_compressedBlockState_t>::zeroed().assume_init() };
|
||||||
|
let state = copy_cctx_internal_test_state(
|
||||||
|
&mut context,
|
||||||
|
0x5000usize as *const c_void,
|
||||||
|
&f_params,
|
||||||
|
&mut destination_block_state_slot,
|
||||||
|
&source_block_state,
|
||||||
|
);
|
||||||
|
|
||||||
let result = unsafe { ZSTD_rust_copyCCtxInternal(&state) };
|
let result = unsafe { ZSTD_rust_copyCCtxInternal(&state) };
|
||||||
|
|
||||||
@@ -13973,8 +14018,19 @@ mod tests {
|
|||||||
..CopyCCtxInternalTestContext::default()
|
..CopyCCtxInternalTestContext::default()
|
||||||
};
|
};
|
||||||
let f_params = ZSTD_frameParameters::default();
|
let f_params = ZSTD_frameParameters::default();
|
||||||
let state =
|
let mut destination_block_state =
|
||||||
copy_cctx_internal_test_state(&mut context, 0x5000usize as *const c_void, &f_params);
|
unsafe { MaybeUninit::<ZSTD_compressedBlockState_t>::zeroed().assume_init() };
|
||||||
|
let mut destination_block_state_slot =
|
||||||
|
&mut destination_block_state as *mut ZSTD_compressedBlockState_t;
|
||||||
|
let source_block_state =
|
||||||
|
unsafe { MaybeUninit::<ZSTD_compressedBlockState_t>::zeroed().assume_init() };
|
||||||
|
let state = copy_cctx_internal_test_state(
|
||||||
|
&mut context,
|
||||||
|
0x5000usize as *const c_void,
|
||||||
|
&f_params,
|
||||||
|
&mut destination_block_state_slot,
|
||||||
|
&source_block_state,
|
||||||
|
);
|
||||||
|
|
||||||
let result = unsafe { ZSTD_rust_copyCCtxInternal(&state) };
|
let result = unsafe { ZSTD_rust_copyCCtxInternal(&state) };
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user