ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move CCtx table copies into Rust

Browse Source 
Pass CCtx match-table locations and source sizing metadata through the Rust
copy bridge. Rust now selects the table sizes after the source-stage guard,
loads destination tables after reset has allocated them, and performs the
hash, chain, and hash3 copies between the existing workspace dirty/clean
callbacks.

Test Plan:
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings
- ulimit -v 41943040 && make -j1
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml
- ulimit -v 41943040 && make -j1 -C tests test-zstream ZSTREAM_TESTTIME=-T2s
- ulimit -v 41943040 && make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests
This commit is contained in:
ddidderr
2026-07-19 23:18:19 +02:00
parent 46dd25a122
commit 21970bdea5
2 changed files with 193 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+31 -30
View File
@@ -462,8 +462,6 @@ typedef size_t (*ZSTD_rust_copyCCtxReset_f)(
const ZSTD_frameParameters* fParams,
U64 pledgedSrcSize, int zbuff);
typedef void (*ZSTD_rust_copyCCtxMarkTables_f)(void* context);
typedef void (*ZSTD_rust_copyCCtxCopyTables_f)(
void* context, const void* srcCCtx);
typedef struct {
void* callbackContext;
const void* srcCCtx;
@@ -474,8 +472,18 @@ typedef struct {
const void* sourceCustomMem;
ZSTD_rust_copyCCtxReset_f reset;
ZSTD_rust_copyCCtxMarkTables_f markTablesDirty;
ZSTD_rust_copyCCtxCopyTables_f copyTables;
ZSTD_rust_copyCCtxMarkTables_f markTablesClean;
U32** destinationHashTable;
const U32* sourceHashTable;
const U32* sourceHashLog;
U32** destinationChainTable;
const U32* sourceChainTable;
const U32* sourceChainLog;
U32** destinationHashTable3;
const U32* sourceHashTable3;
const U32* sourceHashLog3;
const int* sourceStrategy;
const int* sourceUseRowMatchFinder;
void* destinationWindow;
const void* sourceWindow;
U32* destinationNextToUpdate;
@@ -533,7 +541,7 @@ typedef char ZSTD_rust_copy_cctx_internal_state_layout[
== 3 * sizeof(void*) + sizeof(U64)
&& offsetof(ZSTD_rust_copyCCtxInternalState, zbuff)
== 3 * sizeof(void*) + sizeof(U64)
+ 19 * sizeof(void*)
+ 29 * sizeof(void*)
&& sizeof(ZSTD_rust_copyCCtxInternalState)
== ((offsetof(ZSTD_rust_copyCCtxInternalState, zbuff)
+ sizeof(int) + sizeof(void*) - 1) / sizeof(void*))
@@ -4782,31 +4790,6 @@ static void ZSTD_rust_copyCCtx_mark_tables_dirty(void* context)
ZSTD_cwksp_mark_tables_dirty(&dst->workspace);
}
static void ZSTD_rust_copyCCtx_copy_tables(
void* context, const void* srcCCtx)
{
ZSTD_CCtx* const dst = (ZSTD_CCtx*)context;
const ZSTD_CCtx* const src = (const ZSTD_CCtx*)srcCCtx;
size_t const chainSize = ZSTD_allocateChainTable(
src->appliedParams.cParams.strategy,
src->appliedParams.useRowMatchFinder,
0 /* forDDSDict */)
? ((size_t)1 << src->appliedParams.cParams.chainLog)
: 0;
size_t const hSize = (size_t)1 << src->appliedParams.cParams.hashLog;
U32 const h3log = src->blockState.matchState.hashLog3;
size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0;
ZSTD_memcpy(dst->blockState.matchState.hashTable,
src->blockState.matchState.hashTable,
hSize * sizeof(U32));
ZSTD_memcpy(dst->blockState.matchState.chainTable,
src->blockState.matchState.chainTable,
chainSize * sizeof(U32));
ZSTD_memcpy(dst->blockState.matchState.hashTable3,
src->blockState.matchState.hashTable3,
h3Size * sizeof(U32));
}
static void ZSTD_rust_copyCCtx_mark_tables_clean(void* context)
{
ZSTD_CCtx* const dst = (ZSTD_CCtx*)context;
@@ -4819,7 +4802,15 @@ static size_t ZSTD_rust_copyCCtx_internal_callback(
U64 pledgedSrcSize, int zbuff)
{
ZSTD_rust_copyCCtxInternalState state;
ZSTD_CCtx* const dst = (ZSTD_CCtx*)context;
const ZSTD_CCtx* const src = (const ZSTD_CCtx*)srcCCtx;
int const sourceStage = (int)((const ZSTD_CCtx*)srcCCtx)->stage;
U32 const sourceHashLog = src->appliedParams.cParams.hashLog;
U32 const sourceChainLog = src->appliedParams.cParams.chainLog;
U32 const sourceHashLog3 = src->blockState.matchState.hashLog3;
int const sourceStrategy = (int)src->appliedParams.cParams.strategy;
int const sourceUseRowMatchFinder =
(int)src->appliedParams.useRowMatchFinder;
state.callbackContext = context;
state.srcCCtx = srcCCtx;
state.fParams = fParams;
@@ -4831,8 +4822,18 @@ static size_t ZSTD_rust_copyCCtx_internal_callback(
&((const ZSTD_CCtx*)srcCCtx)->customMem;
state.reset = ZSTD_rust_copyCCtx_reset;
state.markTablesDirty = ZSTD_rust_copyCCtx_mark_tables_dirty;
state.copyTables = ZSTD_rust_copyCCtx_copy_tables;
state.markTablesClean = ZSTD_rust_copyCCtx_mark_tables_clean;
state.destinationHashTable = &dst->blockState.matchState.hashTable;
state.sourceHashTable = src->blockState.matchState.hashTable;
state.sourceHashLog = &sourceHashLog;
state.destinationChainTable = &dst->blockState.matchState.chainTable;
state.sourceChainTable = src->blockState.matchState.chainTable;
state.sourceChainLog = &sourceChainLog;
state.destinationHashTable3 = &dst->blockState.matchState.hashTable3;
state.sourceHashTable3 = src->blockState.matchState.hashTable3;
state.sourceHashLog3 = &sourceHashLog3;
state.sourceStrategy = &sourceStrategy;
state.sourceUseRowMatchFinder = &sourceUseRowMatchFinder;
state.destinationWindow =
&((ZSTD_CCtx*)context)->blockState.matchState.window;
state.sourceWindow =
rust/src/zstd_compress.rs
+162 -17
View File
@@ -1680,7 +1680,6 @@ type CopyCCtxResetFn = unsafe extern "C" fn(
c_int,
) -> usize;
type CopyCCtxMarkTablesFn = unsafe extern "C" fn(*mut c_void);
type CopyCCtxCopyTablesFn = unsafe extern "C" fn(*mut c_void, *const c_void);
#[repr(C)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -1733,8 +1732,18 @@ pub struct ZSTD_rust_copyCCtxInternalState {
source_custom_mem: *const c_void,
reset: Option<CopyCCtxResetFn>,
mark_tables_dirty: Option<CopyCCtxMarkTablesFn>,
copy_tables: Option<CopyCCtxCopyTablesFn>,
mark_tables_clean: Option<CopyCCtxMarkTablesFn>,
destination_hash_table: *mut *mut c_uint,
source_hash_table: *const c_uint,
source_hash_log: *const c_uint,
destination_chain_table: *mut *mut c_uint,
source_chain_table: *const c_uint,
source_chain_log: *const c_uint,
destination_hash_table3: *mut *mut c_uint,
source_hash_table3: *const c_uint,
source_hash_log3: *const c_uint,
source_strategy: *const c_int,
source_use_row_match_finder: *const c_int,
destination_window: *mut c_void,
source_window: *const c_void,
destination_next_to_update: *mut c_uint,
@@ -1753,7 +1762,6 @@ pub struct ZSTD_rust_copyCCtxInternalState {
const _: () = {
assert!(size_of::<CopyCCtxResetFn>() == size_of::<usize>());
assert!(size_of::<CopyCCtxMarkTablesFn>() == size_of::<usize>());
assert!(size_of::<CopyCCtxCopyTablesFn>() == size_of::<usize>());
assert!(offset_of!(ZSTD_rust_copyCCtxInternalState, callback_context) == 0);
assert!(offset_of!(ZSTD_rust_copyCCtxInternalState, src_cctx) == size_of::<usize>());
assert!(offset_of!(ZSTD_rust_copyCCtxInternalState, f_params) == 2 * size_of::<usize>());
@@ -1766,7 +1774,7 @@ const _: () = {
);
assert!(
offset_of!(ZSTD_rust_copyCCtxInternalState, zbuff)
== 3 * size_of::<usize>() + size_of::<u64>() + 19 * size_of::<usize>()
== 3 * size_of::<usize>() + size_of::<u64>() + 29 * size_of::<usize>()
);
assert!(
size_of::<ZSTD_rust_copyCCtxInternalState>()
@@ -1786,10 +1794,9 @@ pub unsafe extern "C" fn ZSTD_rust_copyCCtxInternal(
return ERROR(ZstdErrorCode::Generic);
}
let state = unsafe { &*state };
let (Some(reset), Some(mark_tables_dirty), Some(copy_tables), Some(mark_tables_clean)) = (
let (Some(reset), Some(mark_tables_dirty), Some(mark_tables_clean)) = (
state.reset,
state.mark_tables_dirty,
state.copy_tables,
state.mark_tables_clean,
) else {
return ERROR(ZstdErrorCode::Generic);
@@ -1800,6 +1807,14 @@ pub unsafe extern "C" fn ZSTD_rust_copyCCtxInternal(
|| state.source_stage.is_null()
|| state.destination_custom_mem.is_null()
|| state.source_custom_mem.is_null()
|| state.destination_hash_table.is_null()
|| state.destination_chain_table.is_null()
|| state.destination_hash_table3.is_null()
|| state.source_hash_log.is_null()
|| state.source_chain_log.is_null()
|| state.source_hash_log3.is_null()
|| state.source_strategy.is_null()
|| state.source_use_row_match_finder.is_null()
|| state.destination_window.is_null()
|| state.source_window.is_null()
|| state.destination_next_to_update.is_null()
@@ -1820,6 +1835,36 @@ pub unsafe extern "C" fn ZSTD_rust_copyCCtxInternal(
return ERROR(ZstdErrorCode::StageWrong);
}
let source_hash_log = unsafe { *state.source_hash_log };
let source_chain_log = unsafe { *state.source_chain_log };
let source_hash_log3 = unsafe { *state.source_hash_log3 };
let source_strategy = unsafe { *state.source_strategy };
let source_use_row_match_finder = unsafe { *state.source_use_row_match_finder };
let Some(hash_table_size) = 1usize.checked_shl(source_hash_log) else {
return ERROR(ZstdErrorCode::Generic);
};
let chain_table_size = if ZSTD_rust_params_allocateChainTable(
source_strategy,
source_use_row_match_finder,
0,
) != 0
{
let Some(size) = 1usize.checked_shl(source_chain_log) else {
return ERROR(ZstdErrorCode::Generic);
};
size
} else {
0
};
let hash_table3_size = if source_hash_log3 != 0 {
let Some(size) = 1usize.checked_shl(source_hash_log3) else {
return ERROR(ZstdErrorCode::Generic);
};
size
} else {
0
};
unsafe {
ptr::copy(
state.source_custom_mem.cast::<ZSTD_customMem>(),
@@ -1836,8 +1881,40 @@ pub unsafe extern "C" fn ZSTD_rust_copyCCtxInternal(
if ERR_isError(reset_error) {
return reset_error;
}
let destination_hash_table = *state.destination_hash_table;
let destination_chain_table = *state.destination_chain_table;
let destination_hash_table3 = *state.destination_hash_table3;
if (hash_table_size != 0
&& (destination_hash_table.is_null() || state.source_hash_table.is_null()))
|| (chain_table_size != 0
&& (destination_chain_table.is_null() || state.source_chain_table.is_null()))
|| (hash_table3_size != 0
&& (destination_hash_table3.is_null() || state.source_hash_table3.is_null()))
{
return ERROR(ZstdErrorCode::Generic);
}
mark_tables_dirty(state.callback_context);
copy_tables(state.callback_context, state.src_cctx);
if hash_table_size != 0 {
ptr::copy_nonoverlapping(
state.source_hash_table,
destination_hash_table,
hash_table_size,
);
}
if chain_table_size != 0 {
ptr::copy_nonoverlapping(
state.source_chain_table,
destination_chain_table,
chain_table_size,
);
}
if hash_table3_size != 0 {
ptr::copy_nonoverlapping(
state.source_hash_table3,
destination_hash_table3,
hash_table3_size,
);
}
mark_tables_clean(state.callback_context);
ptr::copy(
state.source_window.cast::<ZSTD_rust_copyWindowState>(),
@@ -13923,14 +14000,6 @@ mod tests {
.push("dirty");
}
unsafe extern "C" fn copy_cctx_internal_test_copy_tables(
context: *mut c_void,
_src_cctx: *const c_void,
) {
let context = unsafe { copy_cctx_internal_test_context(context) };
context.events.push("tables");
}
unsafe extern "C" fn copy_cctx_internal_test_mark_tables_clean(context: *mut c_void) {
unsafe { copy_cctx_internal_test_context(context) }
.events
@@ -13944,6 +14013,17 @@ mod tests {
source_stage: *const c_int,
destination_custom_mem: *mut c_void,
source_custom_mem: *const c_void,
destination_hash_table: *mut *mut c_uint,
source_hash_table: *const c_uint,
source_hash_log: *const c_uint,
destination_chain_table: *mut *mut c_uint,
source_chain_table: *const c_uint,
source_chain_log: *const c_uint,
destination_hash_table3: *mut *mut c_uint,
source_hash_table3: *const c_uint,
source_hash_log3: *const c_uint,
source_strategy: *const c_int,
source_use_row_match_finder: *const c_int,
destination_window: *mut c_void,
source_window: *const c_void,
destination_next_to_update: *mut c_uint,
@@ -13967,8 +14047,18 @@ mod tests {
source_custom_mem,
reset: Some(copy_cctx_internal_test_reset),
mark_tables_dirty: Some(copy_cctx_internal_test_mark_tables_dirty),
copy_tables: Some(copy_cctx_internal_test_copy_tables),
mark_tables_clean: Some(copy_cctx_internal_test_mark_tables_clean),
destination_hash_table,
source_hash_table,
source_hash_log,
destination_chain_table,
source_chain_table,
source_chain_log,
destination_hash_table3,
source_hash_table3,
source_hash_log3,
source_strategy,
source_use_row_match_finder,
destination_window,
source_window,
destination_next_to_update,
@@ -13996,6 +14086,20 @@ mod tests {
let source_stage = ZSTD_COMPRESSION_STAGE_INIT;
let source_custom_mem = [0x11usize, 0x22, 0x33];
let mut destination_custom_mem = [0usize; 3];
let source_hash_table = [11u32, 12, 13, 14];
let mut destination_hash_table = [0u32; 4];
let mut destination_hash_table_ptr = destination_hash_table.as_mut_ptr();
let source_hash_log = 2;
let source_chain_table = [21u32, 22];
let mut destination_chain_table = [0u32; 2];
let mut destination_chain_table_ptr = destination_chain_table.as_mut_ptr();
let source_chain_log = 1;
let source_hash_table3 = [31u32, 32];
let mut destination_hash_table3 = [0u32; 2];
let mut destination_hash_table3_ptr = destination_hash_table3.as_mut_ptr();
let source_hash_log3 = 1;
let source_strategy = ZSTD_DFAST;
let source_use_row_match_finder = ZSTD_RUST_PS_DISABLE;
let mut source_block_state =
unsafe { MaybeUninit::<ZSTD_compressedBlockState_t>::zeroed().assume_init() };
source_block_state.rep = [15, 16, 17];
@@ -14036,6 +14140,17 @@ mod tests {
&source_stage,
(&mut destination_custom_mem as *mut [usize; 3]).cast(),
(&source_custom_mem as *const [usize; 3]).cast(),
&mut destination_hash_table_ptr,
source_hash_table.as_ptr(),
&source_hash_log,
&mut destination_chain_table_ptr,
source_chain_table.as_ptr(),
&source_chain_log,
&mut destination_hash_table3_ptr,
source_hash_table3.as_ptr(),
&source_hash_log3,
&source_strategy,
&source_use_row_match_finder,
(&mut destination_window as *mut ZSTD_rust_copyWindowState).cast(),
(&source_window as *const ZSTD_rust_copyWindowState).cast(),
&mut destination_next_to_update,
@@ -14053,11 +14168,14 @@ mod tests {
let result = unsafe { ZSTD_rust_copyCCtxInternal(&state) };
assert_eq!(result, 0);
assert_eq!(context.events, ["reset", "dirty", "tables", "clean"]);
assert_eq!(context.events, ["reset", "dirty", "clean"]);
assert_eq!(context.frame_params, f_params);
assert_eq!(context.pledged_src_size, 123);
assert_eq!(context.zbuff, 7);
assert_eq!(destination_custom_mem, source_custom_mem);
assert_eq!(destination_hash_table, source_hash_table);
assert_eq!(destination_chain_table, source_chain_table);
assert_eq!(destination_hash_table3, source_hash_table3);
assert_eq!(destination_window, source_window);
assert_eq!(destination_next_to_update, source_next_to_update);
assert_eq!(destination_loaded_dict_end, source_loaded_dict_end);
@@ -14100,6 +14218,17 @@ mod tests {
(&source_custom_mem as *const [usize; 3]).cast(),
ptr::dangling_mut(),
ptr::dangling(),
ptr::dangling(),
ptr::dangling_mut(),
ptr::dangling(),
ptr::dangling(),
ptr::dangling_mut(),
ptr::dangling(),
ptr::dangling(),
ptr::dangling(),
ptr::dangling(),
ptr::dangling_mut(),
ptr::dangling(),
ptr::dangling_mut(),
ptr::dangling(),
ptr::dangling_mut(),
@@ -14128,6 +14257,11 @@ mod tests {
let source_stage = ZSTD_COMPRESSION_STAGE_INIT;
let source_custom_mem = [0usize; 3];
let mut destination_custom_mem = [0usize; 3];
let source_hash_log = 0;
let source_chain_log = 0;
let source_hash_log3 = 0;
let source_strategy = ZSTD_DFAST;
let source_use_row_match_finder = ZSTD_RUST_PS_DISABLE;
let mut destination_block_state =
unsafe { MaybeUninit::<ZSTD_compressedBlockState_t>::zeroed().assume_init() };
let mut destination_block_state_slot =
@@ -14147,6 +14281,17 @@ mod tests {
(&source_custom_mem as *const [usize; 3]).cast(),
ptr::dangling_mut(),
ptr::dangling(),
&source_hash_log,
ptr::dangling_mut(),
ptr::dangling(),
&source_chain_log,
ptr::dangling_mut(),
ptr::dangling(),
&source_hash_log3,
&source_strategy,
&source_use_row_match_finder,
ptr::dangling_mut(),
ptr::dangling(),
ptr::dangling_mut(),
ptr::dangling(),
ptr::dangling_mut(),