feat(compress): move CCtx table copies into Rust
Pass CCtx match-table locations and source sizing metadata through the Rust copy bridge. Rust now selects the table sizes after the source-stage guard, loads destination tables after reset has allocated them, and performs the hash, chain, and hash3 copies between the existing workspace dirty/clean callbacks. Test Plan: - ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040 && make -j1 - ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml - ulimit -v 41943040 && make -j1 -C tests test-zstream ZSTREAM_TESTTIME=-T2s - ulimit -v 41943040 && make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests
This commit is contained in:
+162
-17
@@ -1680,7 +1680,6 @@ type CopyCCtxResetFn = unsafe extern "C" fn(
|
||||
c_int,
|
||||
) -> usize;
|
||||
type CopyCCtxMarkTablesFn = unsafe extern "C" fn(*mut c_void);
|
||||
type CopyCCtxCopyTablesFn = unsafe extern "C" fn(*mut c_void, *const c_void);
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
@@ -1733,8 +1732,18 @@ pub struct ZSTD_rust_copyCCtxInternalState {
|
||||
source_custom_mem: *const c_void,
|
||||
reset: Option<CopyCCtxResetFn>,
|
||||
mark_tables_dirty: Option<CopyCCtxMarkTablesFn>,
|
||||
copy_tables: Option<CopyCCtxCopyTablesFn>,
|
||||
mark_tables_clean: Option<CopyCCtxMarkTablesFn>,
|
||||
destination_hash_table: *mut *mut c_uint,
|
||||
source_hash_table: *const c_uint,
|
||||
source_hash_log: *const c_uint,
|
||||
destination_chain_table: *mut *mut c_uint,
|
||||
source_chain_table: *const c_uint,
|
||||
source_chain_log: *const c_uint,
|
||||
destination_hash_table3: *mut *mut c_uint,
|
||||
source_hash_table3: *const c_uint,
|
||||
source_hash_log3: *const c_uint,
|
||||
source_strategy: *const c_int,
|
||||
source_use_row_match_finder: *const c_int,
|
||||
destination_window: *mut c_void,
|
||||
source_window: *const c_void,
|
||||
destination_next_to_update: *mut c_uint,
|
||||
@@ -1753,7 +1762,6 @@ pub struct ZSTD_rust_copyCCtxInternalState {
|
||||
const _: () = {
|
||||
assert!(size_of::<CopyCCtxResetFn>() == size_of::<usize>());
|
||||
assert!(size_of::<CopyCCtxMarkTablesFn>() == size_of::<usize>());
|
||||
assert!(size_of::<CopyCCtxCopyTablesFn>() == size_of::<usize>());
|
||||
assert!(offset_of!(ZSTD_rust_copyCCtxInternalState, callback_context) == 0);
|
||||
assert!(offset_of!(ZSTD_rust_copyCCtxInternalState, src_cctx) == size_of::<usize>());
|
||||
assert!(offset_of!(ZSTD_rust_copyCCtxInternalState, f_params) == 2 * size_of::<usize>());
|
||||
@@ -1766,7 +1774,7 @@ const _: () = {
|
||||
);
|
||||
assert!(
|
||||
offset_of!(ZSTD_rust_copyCCtxInternalState, zbuff)
|
||||
== 3 * size_of::<usize>() + size_of::<u64>() + 19 * size_of::<usize>()
|
||||
== 3 * size_of::<usize>() + size_of::<u64>() + 29 * size_of::<usize>()
|
||||
);
|
||||
assert!(
|
||||
size_of::<ZSTD_rust_copyCCtxInternalState>()
|
||||
@@ -1786,10 +1794,9 @@ pub unsafe extern "C" fn ZSTD_rust_copyCCtxInternal(
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
}
|
||||
let state = unsafe { &*state };
|
||||
let (Some(reset), Some(mark_tables_dirty), Some(copy_tables), Some(mark_tables_clean)) = (
|
||||
let (Some(reset), Some(mark_tables_dirty), Some(mark_tables_clean)) = (
|
||||
state.reset,
|
||||
state.mark_tables_dirty,
|
||||
state.copy_tables,
|
||||
state.mark_tables_clean,
|
||||
) else {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
@@ -1800,6 +1807,14 @@ pub unsafe extern "C" fn ZSTD_rust_copyCCtxInternal(
|
||||
|| state.source_stage.is_null()
|
||||
|| state.destination_custom_mem.is_null()
|
||||
|| state.source_custom_mem.is_null()
|
||||
|| state.destination_hash_table.is_null()
|
||||
|| state.destination_chain_table.is_null()
|
||||
|| state.destination_hash_table3.is_null()
|
||||
|| state.source_hash_log.is_null()
|
||||
|| state.source_chain_log.is_null()
|
||||
|| state.source_hash_log3.is_null()
|
||||
|| state.source_strategy.is_null()
|
||||
|| state.source_use_row_match_finder.is_null()
|
||||
|| state.destination_window.is_null()
|
||||
|| state.source_window.is_null()
|
||||
|| state.destination_next_to_update.is_null()
|
||||
@@ -1820,6 +1835,36 @@ pub unsafe extern "C" fn ZSTD_rust_copyCCtxInternal(
|
||||
return ERROR(ZstdErrorCode::StageWrong);
|
||||
}
|
||||
|
||||
let source_hash_log = unsafe { *state.source_hash_log };
|
||||
let source_chain_log = unsafe { *state.source_chain_log };
|
||||
let source_hash_log3 = unsafe { *state.source_hash_log3 };
|
||||
let source_strategy = unsafe { *state.source_strategy };
|
||||
let source_use_row_match_finder = unsafe { *state.source_use_row_match_finder };
|
||||
let Some(hash_table_size) = 1usize.checked_shl(source_hash_log) else {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
};
|
||||
let chain_table_size = if ZSTD_rust_params_allocateChainTable(
|
||||
source_strategy,
|
||||
source_use_row_match_finder,
|
||||
0,
|
||||
) != 0
|
||||
{
|
||||
let Some(size) = 1usize.checked_shl(source_chain_log) else {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
};
|
||||
size
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let hash_table3_size = if source_hash_log3 != 0 {
|
||||
let Some(size) = 1usize.checked_shl(source_hash_log3) else {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
};
|
||||
size
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
unsafe {
|
||||
ptr::copy(
|
||||
state.source_custom_mem.cast::<ZSTD_customMem>(),
|
||||
@@ -1836,8 +1881,40 @@ pub unsafe extern "C" fn ZSTD_rust_copyCCtxInternal(
|
||||
if ERR_isError(reset_error) {
|
||||
return reset_error;
|
||||
}
|
||||
let destination_hash_table = *state.destination_hash_table;
|
||||
let destination_chain_table = *state.destination_chain_table;
|
||||
let destination_hash_table3 = *state.destination_hash_table3;
|
||||
if (hash_table_size != 0
|
||||
&& (destination_hash_table.is_null() || state.source_hash_table.is_null()))
|
||||
|| (chain_table_size != 0
|
||||
&& (destination_chain_table.is_null() || state.source_chain_table.is_null()))
|
||||
|| (hash_table3_size != 0
|
||||
&& (destination_hash_table3.is_null() || state.source_hash_table3.is_null()))
|
||||
{
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
}
|
||||
mark_tables_dirty(state.callback_context);
|
||||
copy_tables(state.callback_context, state.src_cctx);
|
||||
if hash_table_size != 0 {
|
||||
ptr::copy_nonoverlapping(
|
||||
state.source_hash_table,
|
||||
destination_hash_table,
|
||||
hash_table_size,
|
||||
);
|
||||
}
|
||||
if chain_table_size != 0 {
|
||||
ptr::copy_nonoverlapping(
|
||||
state.source_chain_table,
|
||||
destination_chain_table,
|
||||
chain_table_size,
|
||||
);
|
||||
}
|
||||
if hash_table3_size != 0 {
|
||||
ptr::copy_nonoverlapping(
|
||||
state.source_hash_table3,
|
||||
destination_hash_table3,
|
||||
hash_table3_size,
|
||||
);
|
||||
}
|
||||
mark_tables_clean(state.callback_context);
|
||||
ptr::copy(
|
||||
state.source_window.cast::<ZSTD_rust_copyWindowState>(),
|
||||
@@ -13923,14 +14000,6 @@ mod tests {
|
||||
.push("dirty");
|
||||
}
|
||||
|
||||
unsafe extern "C" fn copy_cctx_internal_test_copy_tables(
|
||||
context: *mut c_void,
|
||||
_src_cctx: *const c_void,
|
||||
) {
|
||||
let context = unsafe { copy_cctx_internal_test_context(context) };
|
||||
context.events.push("tables");
|
||||
}
|
||||
|
||||
unsafe extern "C" fn copy_cctx_internal_test_mark_tables_clean(context: *mut c_void) {
|
||||
unsafe { copy_cctx_internal_test_context(context) }
|
||||
.events
|
||||
@@ -13944,6 +14013,17 @@ mod tests {
|
||||
source_stage: *const c_int,
|
||||
destination_custom_mem: *mut c_void,
|
||||
source_custom_mem: *const c_void,
|
||||
destination_hash_table: *mut *mut c_uint,
|
||||
source_hash_table: *const c_uint,
|
||||
source_hash_log: *const c_uint,
|
||||
destination_chain_table: *mut *mut c_uint,
|
||||
source_chain_table: *const c_uint,
|
||||
source_chain_log: *const c_uint,
|
||||
destination_hash_table3: *mut *mut c_uint,
|
||||
source_hash_table3: *const c_uint,
|
||||
source_hash_log3: *const c_uint,
|
||||
source_strategy: *const c_int,
|
||||
source_use_row_match_finder: *const c_int,
|
||||
destination_window: *mut c_void,
|
||||
source_window: *const c_void,
|
||||
destination_next_to_update: *mut c_uint,
|
||||
@@ -13967,8 +14047,18 @@ mod tests {
|
||||
source_custom_mem,
|
||||
reset: Some(copy_cctx_internal_test_reset),
|
||||
mark_tables_dirty: Some(copy_cctx_internal_test_mark_tables_dirty),
|
||||
copy_tables: Some(copy_cctx_internal_test_copy_tables),
|
||||
mark_tables_clean: Some(copy_cctx_internal_test_mark_tables_clean),
|
||||
destination_hash_table,
|
||||
source_hash_table,
|
||||
source_hash_log,
|
||||
destination_chain_table,
|
||||
source_chain_table,
|
||||
source_chain_log,
|
||||
destination_hash_table3,
|
||||
source_hash_table3,
|
||||
source_hash_log3,
|
||||
source_strategy,
|
||||
source_use_row_match_finder,
|
||||
destination_window,
|
||||
source_window,
|
||||
destination_next_to_update,
|
||||
@@ -13996,6 +14086,20 @@ mod tests {
|
||||
let source_stage = ZSTD_COMPRESSION_STAGE_INIT;
|
||||
let source_custom_mem = [0x11usize, 0x22, 0x33];
|
||||
let mut destination_custom_mem = [0usize; 3];
|
||||
let source_hash_table = [11u32, 12, 13, 14];
|
||||
let mut destination_hash_table = [0u32; 4];
|
||||
let mut destination_hash_table_ptr = destination_hash_table.as_mut_ptr();
|
||||
let source_hash_log = 2;
|
||||
let source_chain_table = [21u32, 22];
|
||||
let mut destination_chain_table = [0u32; 2];
|
||||
let mut destination_chain_table_ptr = destination_chain_table.as_mut_ptr();
|
||||
let source_chain_log = 1;
|
||||
let source_hash_table3 = [31u32, 32];
|
||||
let mut destination_hash_table3 = [0u32; 2];
|
||||
let mut destination_hash_table3_ptr = destination_hash_table3.as_mut_ptr();
|
||||
let source_hash_log3 = 1;
|
||||
let source_strategy = ZSTD_DFAST;
|
||||
let source_use_row_match_finder = ZSTD_RUST_PS_DISABLE;
|
||||
let mut source_block_state =
|
||||
unsafe { MaybeUninit::<ZSTD_compressedBlockState_t>::zeroed().assume_init() };
|
||||
source_block_state.rep = [15, 16, 17];
|
||||
@@ -14036,6 +14140,17 @@ mod tests {
|
||||
&source_stage,
|
||||
(&mut destination_custom_mem as *mut [usize; 3]).cast(),
|
||||
(&source_custom_mem as *const [usize; 3]).cast(),
|
||||
&mut destination_hash_table_ptr,
|
||||
source_hash_table.as_ptr(),
|
||||
&source_hash_log,
|
||||
&mut destination_chain_table_ptr,
|
||||
source_chain_table.as_ptr(),
|
||||
&source_chain_log,
|
||||
&mut destination_hash_table3_ptr,
|
||||
source_hash_table3.as_ptr(),
|
||||
&source_hash_log3,
|
||||
&source_strategy,
|
||||
&source_use_row_match_finder,
|
||||
(&mut destination_window as *mut ZSTD_rust_copyWindowState).cast(),
|
||||
(&source_window as *const ZSTD_rust_copyWindowState).cast(),
|
||||
&mut destination_next_to_update,
|
||||
@@ -14053,11 +14168,14 @@ mod tests {
|
||||
let result = unsafe { ZSTD_rust_copyCCtxInternal(&state) };
|
||||
|
||||
assert_eq!(result, 0);
|
||||
assert_eq!(context.events, ["reset", "dirty", "tables", "clean"]);
|
||||
assert_eq!(context.events, ["reset", "dirty", "clean"]);
|
||||
assert_eq!(context.frame_params, f_params);
|
||||
assert_eq!(context.pledged_src_size, 123);
|
||||
assert_eq!(context.zbuff, 7);
|
||||
assert_eq!(destination_custom_mem, source_custom_mem);
|
||||
assert_eq!(destination_hash_table, source_hash_table);
|
||||
assert_eq!(destination_chain_table, source_chain_table);
|
||||
assert_eq!(destination_hash_table3, source_hash_table3);
|
||||
assert_eq!(destination_window, source_window);
|
||||
assert_eq!(destination_next_to_update, source_next_to_update);
|
||||
assert_eq!(destination_loaded_dict_end, source_loaded_dict_end);
|
||||
@@ -14100,6 +14218,17 @@ mod tests {
|
||||
(&source_custom_mem as *const [usize; 3]).cast(),
|
||||
ptr::dangling_mut(),
|
||||
ptr::dangling(),
|
||||
ptr::dangling(),
|
||||
ptr::dangling_mut(),
|
||||
ptr::dangling(),
|
||||
ptr::dangling(),
|
||||
ptr::dangling_mut(),
|
||||
ptr::dangling(),
|
||||
ptr::dangling(),
|
||||
ptr::dangling(),
|
||||
ptr::dangling(),
|
||||
ptr::dangling_mut(),
|
||||
ptr::dangling(),
|
||||
ptr::dangling_mut(),
|
||||
ptr::dangling(),
|
||||
ptr::dangling_mut(),
|
||||
@@ -14128,6 +14257,11 @@ mod tests {
|
||||
let source_stage = ZSTD_COMPRESSION_STAGE_INIT;
|
||||
let source_custom_mem = [0usize; 3];
|
||||
let mut destination_custom_mem = [0usize; 3];
|
||||
let source_hash_log = 0;
|
||||
let source_chain_log = 0;
|
||||
let source_hash_log3 = 0;
|
||||
let source_strategy = ZSTD_DFAST;
|
||||
let source_use_row_match_finder = ZSTD_RUST_PS_DISABLE;
|
||||
let mut destination_block_state =
|
||||
unsafe { MaybeUninit::<ZSTD_compressedBlockState_t>::zeroed().assume_init() };
|
||||
let mut destination_block_state_slot =
|
||||
@@ -14147,6 +14281,17 @@ mod tests {
|
||||
(&source_custom_mem as *const [usize; 3]).cast(),
|
||||
ptr::dangling_mut(),
|
||||
ptr::dangling(),
|
||||
&source_hash_log,
|
||||
ptr::dangling_mut(),
|
||||
ptr::dangling(),
|
||||
&source_chain_log,
|
||||
ptr::dangling_mut(),
|
||||
ptr::dangling(),
|
||||
&source_hash_log3,
|
||||
&source_strategy,
|
||||
&source_use_row_match_finder,
|
||||
ptr::dangling_mut(),
|
||||
ptr::dangling(),
|
||||
ptr::dangling_mut(),
|
||||
ptr::dangling(),
|
||||
ptr::dangling_mut(),
|
||||
|
||||
Reference in New Issue
Block a user