ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move workspace resize policy to Rust

Browse Source 
The CCtx reset orchestration already runs in Rust, but its C workspace
adapter still decided when capacity was too small or wasteful and maintained
the oversized-duration counter through C-only predicates. Move those factor,
threshold, overflow, and duration-update rules into Rust. C now supplies only
private workspace metrics and retains low-level allocation, reservation, and
layout operations.

Remove the obsolete C predicates so the workspace resize decision has one
Rust-owned implementation and remains covered at its boundary conditions.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo check --manifest-path rust/Cargo.toml --tests
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 make -j1
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 make -j1 -C tests invalidDictionaries
- ulimit -v 41943040; make -j1 -C tests test
This commit is contained in:
ddidderr
2026-07-21 17:55:20 +02:00
parent 49a448602a
commit e3ce950f01
3 changed files with 114 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+14 -3
View File
@@ -2166,6 +2166,13 @@ typedef void (*ZSTD_rust_resetCCtxWorkspacePrepare_f)(
size_t blockSize, int* workspaceTooSmall, int* workspaceWasteful);
typedef void (*ZSTD_rust_resetCCtxTailPrepare_f)(
void* context, size_t blockSize, int indexResetPolicy);
void ZSTD_rust_workspaceResizeFlags(
size_t workspaceSize, size_t availableSpace,
int workspaceOversizedDuration, size_t neededSpace,
int* workspaceTooSmall, int* workspaceWasteful);
void ZSTD_rust_workspaceBumpOversizedDuration(
size_t availableSpace, size_t additionalNeededSpace,
int* workspaceOversizedDuration);
typedef struct {
void* callbackContext;
int ldmEnable;
@@ -5270,7 +5277,9 @@ static void ZSTD_rust_resetCCtxWorkspace_bumpOversizedDuration(void* opaque)
{
ZSTD_rust_resetCCtxStorageContext* const context =
(ZSTD_rust_resetCCtxStorageContext*)opaque;
ZSTD_cwksp_bump_oversized_duration(context->ws, 0);
ZSTD_rust_workspaceBumpOversizedDuration(
ZSTD_cwksp_available_space(context->ws), 0,
&context->ws->workspaceOversizedDuration);
}
static void ZSTD_rust_resetCCtxWorkspace_free(void* opaque)
@@ -5321,8 +5330,10 @@ static void ZSTD_rust_resetCCtxWorkspace_prepare(
ZSTD_rust_resetCCtxTailContext* const context =
(ZSTD_rust_resetCCtxTailContext*)opaque;
ZSTD_cwksp* const ws = context->ws;
*workspaceTooSmall = ZSTD_cwksp_sizeof(ws) < neededSpace;
*workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace);
ZSTD_rust_workspaceResizeFlags(
ZSTD_cwksp_sizeof(ws), ZSTD_cwksp_available_space(ws),
ws->workspaceOversizedDuration, neededSpace,
workspaceTooSmall, workspaceWasteful);
DEBUGLOG(4, "Need %zu B workspace", neededSpace);
DEBUGLOG(4, "windowSize: %zu - blockSize: %zu",
windowSize, blockSize);
lib/compress/zstd_cwksp.h
-19
View File
@@ -743,23 +743,4 @@ MEM_STATIC int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeede
return ZSTD_cwksp_available_space(ws) >= additionalNeededSpace;
}
MEM_STATIC int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace) {
return ZSTD_cwksp_check_available(
ws, additionalNeededSpace * ZSTD_WORKSPACETOOLARGE_FACTOR);
}
MEM_STATIC int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace) {
return ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)
&& ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION;
}
MEM_STATIC void ZSTD_cwksp_bump_oversized_duration(
ZSTD_cwksp* ws, size_t additionalNeededSpace) {
if (ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)) {
ws->workspaceOversizedDuration++;
} else {
ws->workspaceOversizedDuration = 0;
}
}
#endif /* ZSTD_CWKSP_H */
rust/src/zstd_compress.rs
+100 -1
View File
@@ -9408,6 +9408,95 @@ type ResetCCtxWorkspaceCreate = unsafe extern "C" fn(*mut c_void, usize) -> usiz
type ResetCCtxWorkspaceReserveObject = unsafe extern "C" fn(*mut c_void, usize) -> *mut c_void;
type ResetCCtxTailCallback = unsafe extern "C" fn(*mut c_void) -> usize;
const ZSTD_WORKSPACETOOLARGE_FACTOR: usize = 3;
const ZSTD_WORKSPACETOOLARGE_MAXDURATION: c_int = 128;
#[inline]
fn workspace_resize_flags(
workspace_size: usize,
available_space: usize,
workspace_oversized_duration: c_int,
needed_space: usize,
) -> (c_int, c_int) {
let workspace_too_small = if workspace_size < needed_space { 1 } else { 0 };
let workspace_too_large = workspace_is_too_large(available_space, needed_space);
let workspace_wasteful = if workspace_too_large
&& workspace_oversized_duration > ZSTD_WORKSPACETOOLARGE_MAXDURATION
{
1
} else {
0
};
(workspace_too_small, workspace_wasteful)
}
#[inline]
fn workspace_is_too_large(available_space: usize, additional_needed_space: usize) -> bool {
available_space
>= additional_needed_space.wrapping_mul(ZSTD_WORKSPACETOOLARGE_FACTOR)
}
#[inline]
fn workspace_bump_oversized_duration(
available_space: usize,
additional_needed_space: usize,
workspace_oversized_duration: c_int,
) -> c_int {
if workspace_is_too_large(available_space, additional_needed_space) {
workspace_oversized_duration.wrapping_add(1)
} else {
0
}
}
/// Decide whether the existing CCtx workspace should be resized.
///
/// C retains the private workspace representation and passes its metrics;
/// Rust owns the resize predicate and its factor/duration policy.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_workspaceResizeFlags(
workspace_size: usize,
available_space: usize,
workspace_oversized_duration: c_int,
needed_space: usize,
workspace_too_small: *mut c_int,
workspace_wasteful: *mut c_int,
) {
if workspace_too_small.is_null() || workspace_wasteful.is_null() {
return;
}
let (too_small, wasteful) = workspace_resize_flags(
workspace_size,
available_space,
workspace_oversized_duration,
needed_space,
);
unsafe {
*workspace_too_small = too_small;
*workspace_wasteful = wasteful;
}
}
/// Update the workspace oversizing duration using the same policy as the
/// resize predicate.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_workspaceBumpOversizedDuration(
available_space: usize,
additional_needed_space: usize,
workspace_oversized_duration: *mut c_int,
) {
if workspace_oversized_duration.is_null() {
return;
}
unsafe {
*workspace_oversized_duration = workspace_bump_oversized_duration(
available_space,
additional_needed_space,
*workspace_oversized_duration,
);
}
}
/// C-owned workspace operations used by the CCtx reset policy.
///
/// Rust owns the resize decision, error ordering, object-reservation order,
@@ -9580,7 +9669,7 @@ type ResetCCtxTailPrepare = unsafe extern "C" fn(*mut c_void, usize, c_int);
/// State for composing the already-projected CCtx reset policies.
///
/// Rust owns the plan/workspace/tail order and the scalar hand-off between
/// those helpers. C retains the workspace checks, context initialization,
/// those helpers. C retains workspace metrics, context initialization,
/// match-state reset, and private pointer publication behind callbacks.
#[repr(C)]
pub struct ZSTD_rust_resetCCtxInternalState {
@@ -22547,6 +22636,16 @@ mod tests {
);
}
#[test]
fn workspace_resize_flags_match_size_and_duration_policy() {
assert_eq!(workspace_resize_flags(1023, 3072, 129, 1024), (1, 1));
assert_eq!(workspace_resize_flags(4096, 3072, 129, 1024), (0, 1));
assert_eq!(workspace_resize_flags(4096, 3072, 128, 1024), (0, 0));
assert_eq!(workspace_resize_flags(4096, 3071, 129, 1024), (0, 0));
assert_eq!(workspace_bump_oversized_duration(0, 0, 128), 129);
assert_eq!(workspace_bump_oversized_duration(2, 1, 129), 0);
}
#[derive(Debug, PartialEq)]
enum ResetCCtxWorkspaceTestEvent {
BumpOversizedDuration,