ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(compress): move CDict construction policy to Rust

Browse Source 
Move advanced and static CDict construction ordering into Rust while keeping
private workspace layout, object initialization, dictionary loading, and
allocator leaves in C callbacks.  The Rust policy now validates parameters and
memory, computes the workspace contract, sequences allocation/reservation and
initialization, and preserves failure cleanup for both dynamic and static
workspaces.  Add callback-order, allocation-failure, reservation-failure, and
initializer-failure tests around the ABI projections.

Test Plan:
- `cargo fmt --manifest-path rust/Cargo.toml --all -- --check`
- `ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --all-targets`
- `ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings`
- `ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo build --manifest-path rust/Cargo.toml --release`
This commit is contained in:
ddidderr
2026-07-20 04:05:14 +02:00
parent 371451d69c
commit 1c3bf6b708
2 changed files with 414 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+164 -79
View File
@@ -404,12 +404,23 @@ typedef char ZSTD_rust_free_cdict_state_layout[
== 3 * sizeof(void*)
&& sizeof(ZSTD_rust_freeCDictState) == 4 * sizeof(void*))
? 1 : -1];
typedef void* (*ZSTD_rust_initStaticCDictInit_f)(void* context);
typedef void (*ZSTD_rust_initStaticCDictCreateWorkspace_f)(void* context);
typedef void* (*ZSTD_rust_initStaticCDictReserveObject_f)(void* context);
typedef void (*ZSTD_rust_initStaticCDictMoveWorkspace_f)(
void* context, void* cdict);
typedef void (*ZSTD_rust_initStaticCDictInitialize_f)(
void* context, void* cdict);
typedef void* (*ZSTD_rust_initStaticCDictInit_f)(
void* context, void* cdict);
typedef struct {
void* callbackContext;
void* workspace;
size_t workspaceSize;
size_t neededSize;
ZSTD_rust_initStaticCDictCreateWorkspace_f createWorkspace;
ZSTD_rust_initStaticCDictReserveObject_f reserveObject;
ZSTD_rust_initStaticCDictMoveWorkspace_f moveWorkspace;
ZSTD_rust_initStaticCDictInitialize_f initialize;
ZSTD_rust_initStaticCDictInit_f init;
} ZSTD_rust_initStaticCDictState;
void* ZSTD_rust_initStaticCDict(const ZSTD_rust_initStaticCDictState* state);
@@ -421,9 +432,17 @@ typedef char ZSTD_rust_init_static_cdict_state_layout[
== 2 * sizeof(void*)
&& offsetof(ZSTD_rust_initStaticCDictState, neededSize)
== 3 * sizeof(void*)
&& offsetof(ZSTD_rust_initStaticCDictState, init)
&& offsetof(ZSTD_rust_initStaticCDictState, createWorkspace)
== 4 * sizeof(void*)
&& sizeof(ZSTD_rust_initStaticCDictState) == 5 * sizeof(void*))
&& offsetof(ZSTD_rust_initStaticCDictState, reserveObject)
== 5 * sizeof(void*)
&& offsetof(ZSTD_rust_initStaticCDictState, moveWorkspace)
== 6 * sizeof(void*)
&& offsetof(ZSTD_rust_initStaticCDictState, initialize)
== 7 * sizeof(void*)
&& offsetof(ZSTD_rust_initStaticCDictState, init)
== 8 * sizeof(void*)
&& sizeof(ZSTD_rust_initStaticCDictState) == 9 * sizeof(void*))
? 1 : -1];
typedef void (*ZSTD_rust_resetCCtxClearAllDicts_f)(void* context);
typedef size_t (*ZSTD_rust_resetCCtxResetParams_f)(void* context);
@@ -2531,11 +2550,14 @@ typedef size_t (*ZSTD_rust_createCDictAdvancedWorkspaceSize_f)(
int useRowMatchFinder, int enableDedicatedDictSearch);
typedef void* (*ZSTD_rust_createCDictAdvancedAllocate_f)(
void* context, size_t workspaceSize);
typedef void* (*ZSTD_rust_createCDictAdvancedCreate_f)(
void* context, void* workspace, size_t workspaceSize,
size_t dictSize, int dictLoadMethod,
const ZSTD_compressionParameters* cParams,
int useRowMatchFinder, int enableDedicatedDictSearch);
typedef void (*ZSTD_rust_createCDictAdvancedCreateWorkspace_f)(
void* context, void* workspace, size_t workspaceSize);
typedef void* (*ZSTD_rust_createCDictAdvancedReserveObject_f)(
void* context);
typedef void (*ZSTD_rust_createCDictAdvancedMoveWorkspace_f)(
void* context, void* cdict);
typedef void (*ZSTD_rust_createCDictAdvancedInitialize_f)(
void* context, void* cdict, int useRowMatchFinder);
typedef size_t (*ZSTD_rust_createCDictAdvancedInit_f)(
void* context, void* cdict, const void* dict, size_t dictSize,
int dictLoadMethod, int dictContentType,
@@ -2555,7 +2577,10 @@ typedef struct {
ZSTD_rust_createCDictAdvancedValidateCustomMem_f validateCustomMem;
ZSTD_rust_createCDictAdvancedWorkspaceSize_f workspaceSize;
ZSTD_rust_createCDictAdvancedAllocate_f allocate;
ZSTD_rust_createCDictAdvancedCreate_f create;
ZSTD_rust_createCDictAdvancedCreateWorkspace_f createWorkspace;
ZSTD_rust_createCDictAdvancedReserveObject_f reserveObject;
ZSTD_rust_createCDictAdvancedMoveWorkspace_f moveWorkspace;
ZSTD_rust_createCDictAdvancedInitialize_f initialize;
ZSTD_rust_createCDictAdvancedInit_f init;
ZSTD_rust_createCDictAdvancedFreeWorkspace_f freeWorkspace;
ZSTD_rust_createCDictAdvancedFree_f free;
@@ -2584,16 +2609,22 @@ typedef char ZSTD_rust_create_cdict_advanced_state_layout[
== 6 * sizeof(void*) + 2 * sizeof(U32)
&& offsetof(ZSTD_rust_createCDictAdvancedState, allocate)
== 7 * sizeof(void*) + 2 * sizeof(U32)
&& offsetof(ZSTD_rust_createCDictAdvancedState, create)
&& offsetof(ZSTD_rust_createCDictAdvancedState, createWorkspace)
== 8 * sizeof(void*) + 2 * sizeof(U32)
&& offsetof(ZSTD_rust_createCDictAdvancedState, init)
&& offsetof(ZSTD_rust_createCDictAdvancedState, reserveObject)
== 9 * sizeof(void*) + 2 * sizeof(U32)
&& offsetof(ZSTD_rust_createCDictAdvancedState, freeWorkspace)
&& offsetof(ZSTD_rust_createCDictAdvancedState, moveWorkspace)
== 10 * sizeof(void*) + 2 * sizeof(U32)
&& offsetof(ZSTD_rust_createCDictAdvancedState, free)
&& offsetof(ZSTD_rust_createCDictAdvancedState, initialize)
== 11 * sizeof(void*) + 2 * sizeof(U32)
&& offsetof(ZSTD_rust_createCDictAdvancedState, init)
== 12 * sizeof(void*) + 2 * sizeof(U32)
&& offsetof(ZSTD_rust_createCDictAdvancedState, freeWorkspace)
== 13 * sizeof(void*) + 2 * sizeof(U32)
&& offsetof(ZSTD_rust_createCDictAdvancedState, free)
== 14 * sizeof(void*) + 2 * sizeof(U32)
&& sizeof(ZSTD_rust_createCDictAdvancedState)
== 12 * sizeof(void*) + 2 * sizeof(U32))
== 15 * sizeof(void*) + 2 * sizeof(U32))
? 1 : -1];
typedef size_t (*ZSTD_rust_compressBeginResetInternal_f)(
@@ -6710,9 +6741,16 @@ static size_t ZSTD_initCDict_internal(
(int)dictLoadMethod, (int)dictContentType);
}
typedef struct {
ZSTD_customMem customMem;
ZSTD_cwksp workspace;
} ZSTD_rust_createCDictAdvancedContext;
static int ZSTD_rust_createCDictAdvanced_validateCustomMem(void* context)
{
ZSTD_customMem const* const customMem = (const ZSTD_customMem*)context;
ZSTD_rust_createCDictAdvancedContext const* const createContext =
(const ZSTD_rust_createCDictAdvancedContext*)context;
ZSTD_customMem const* const customMem = &createContext->customMem;
return ((!customMem->customAlloc) ^ (!customMem->customFree)) == 0;
}
@@ -6741,59 +6779,54 @@ static size_t ZSTD_rust_createCDictAdvanced_workspaceSize(
static void* ZSTD_rust_createCDictAdvanced_allocate(
void* context, size_t workspaceSize)
{
return ZSTD_customMalloc(workspaceSize, *(const ZSTD_customMem*)context);
ZSTD_rust_createCDictAdvancedContext const* const createContext =
(const ZSTD_rust_createCDictAdvancedContext*)context;
return ZSTD_customMalloc(workspaceSize, createContext->customMem);
}
static void* ZSTD_rust_createCDictAdvanced_create(
void* context, void* workspace, size_t workspaceSize,
size_t dictSize, int dictLoadMethod,
const ZSTD_compressionParameters* cParams,
int useRowMatchFinder, int enableDedicatedDictSearch)
static void ZSTD_rust_createCDictAdvanced_createWorkspace(
void* context, void* workspace, size_t workspaceSize)
{
ZSTD_customMem const customMem = *(const ZSTD_customMem*)context;
ZSTD_cwksp ws;
ZSTD_CDict* cdict;
(void)dictSize;
(void)dictLoadMethod;
(void)cParams;
(void)enableDedicatedDictSearch;
ZSTD_rust_createCDictAdvancedContext* const createContext =
(ZSTD_rust_createCDictAdvancedContext*)context;
DEBUGLOG(3, "ZSTD_createCDict_advanced_internal (workspaceSize=%u)",
(unsigned)workspaceSize);
if (workspace == NULL) return NULL;
ZSTD_cwksp_init(&createContext->workspace, workspace, workspaceSize,
ZSTD_cwksp_dynamic_alloc);
}
ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_dynamic_alloc);
cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict));
if (cdict == NULL) return NULL;
ZSTD_cwksp_move(&cdict->workspace, &ws);
cdict->customMem = customMem;
cdict->compressionLevel = ZSTD_NO_CLEVEL; /* signals advanced API usage */
cdict->useRowMatchFinder = (ZSTD_ParamSwitch_e)useRowMatchFinder;
return cdict;
static void* ZSTD_rust_createCDictAdvanced_reserveObject(void* context)
{
ZSTD_rust_createCDictAdvancedContext* const createContext =
(ZSTD_rust_createCDictAdvancedContext*)context;
return ZSTD_cwksp_reserve_object(&createContext->workspace, sizeof(ZSTD_CDict));
}
static void ZSTD_rust_createCDictAdvanced_moveWorkspace(
void* context, void* cdict)
{
ZSTD_rust_createCDictAdvancedContext* const createContext =
(ZSTD_rust_createCDictAdvancedContext*)context;
ZSTD_cwksp_move(&((ZSTD_CDict*)cdict)->workspace, &createContext->workspace);
}
static void ZSTD_rust_createCDictAdvanced_initialize(
void* context, void* cdict, int useRowMatchFinder)
{
ZSTD_rust_createCDictAdvancedContext const* const createContext =
(const ZSTD_rust_createCDictAdvancedContext*)context;
ZSTD_CDict* const dictionary = (ZSTD_CDict*)cdict;
dictionary->customMem = createContext->customMem;
dictionary->compressionLevel = ZSTD_NO_CLEVEL; /* signals advanced API usage */
dictionary->useRowMatchFinder = (ZSTD_ParamSwitch_e)useRowMatchFinder;
}
static void ZSTD_rust_createCDictAdvanced_freeWorkspace(
void* context, void* workspace)
{
ZSTD_customFree(workspace, *(const ZSTD_customMem*)context);
}
ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType,
ZSTD_compressionParameters cParams,
ZSTD_customMem customMem)
{
ZSTD_CCtx_params cctxParams;
ZSTD_memset(&cctxParams, 0, sizeof(cctxParams));
DEBUGLOG(3, "ZSTD_createCDict_advanced, dictSize=%u, mode=%u", (unsigned)dictSize, (unsigned)dictContentType);
ZSTD_CCtxParams_init(&cctxParams, 0);
cctxParams.cParams = cParams;
cctxParams.customMem = customMem;
return ZSTD_createCDict_advanced2(
dictBuffer, dictSize,
dictLoadMethod, dictContentType,
&cctxParams, customMem);
ZSTD_rust_createCDictAdvancedContext const* const createContext =
(const ZSTD_rust_createCDictAdvancedContext*)context;
ZSTD_customFree(workspace, createContext->customMem);
}
static size_t ZSTD_rust_createCDictAdvanced_init(
@@ -6814,6 +6847,24 @@ static void ZSTD_rust_createCDictAdvanced_free(void* context, void* cdict)
ZSTD_freeCDict((ZSTD_CDict*)cdict);
}
ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType,
ZSTD_compressionParameters cParams,
ZSTD_customMem customMem)
{
ZSTD_CCtx_params cctxParams;
ZSTD_memset(&cctxParams, 0, sizeof(cctxParams));
DEBUGLOG(3, "ZSTD_createCDict_advanced, dictSize=%u, mode=%u", (unsigned)dictSize, (unsigned)dictContentType);
ZSTD_CCtxParams_init(&cctxParams, 0);
cctxParams.cParams = cParams;
cctxParams.customMem = customMem;
return ZSTD_createCDict_advanced2(
dictBuffer, dictSize,
dictLoadMethod, dictContentType,
&cctxParams, customMem);
}
ZSTD_CDict* ZSTD_createCDict_advanced2(
const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
@@ -6822,13 +6873,15 @@ ZSTD_CDict* ZSTD_createCDict_advanced2(
ZSTD_customMem customMem)
{
ZSTD_CCtx_params cctxParams;
ZSTD_rust_createCDictAdvancedContext context;
ZSTD_rust_createCDictAdvancedState state;
DEBUGLOG(3, "ZSTD_createCDict_advanced2, dictSize=%u, mode=%u", (unsigned)dictSize, (unsigned)dictContentType);
if (originalCctxParams == NULL) return NULL;
cctxParams = *originalCctxParams;
state.callbackContext = &customMem;
context.customMem = customMem;
state.callbackContext = &context;
state.cctxParams = &cctxParams;
state.cParams = &cctxParams.cParams;
state.enableDedicatedDictSearch = &cctxParams.enableDedicatedDictSearch;
@@ -6838,7 +6891,10 @@ ZSTD_CDict* ZSTD_createCDict_advanced2(
state.validateCustomMem = ZSTD_rust_createCDictAdvanced_validateCustomMem;
state.workspaceSize = ZSTD_rust_createCDictAdvanced_workspaceSize;
state.allocate = ZSTD_rust_createCDictAdvanced_allocate;
state.create = ZSTD_rust_createCDictAdvanced_create;
state.createWorkspace = ZSTD_rust_createCDictAdvanced_createWorkspace;
state.reserveObject = ZSTD_rust_createCDictAdvanced_reserveObject;
state.moveWorkspace = ZSTD_rust_createCDictAdvanced_moveWorkspace;
state.initialize = ZSTD_rust_createCDictAdvanced_initialize;
state.init = ZSTD_rust_createCDictAdvanced_init;
state.freeWorkspace = ZSTD_rust_createCDictAdvanced_freeWorkspace;
state.free = ZSTD_rust_createCDictAdvanced_free;
@@ -6847,6 +6903,10 @@ ZSTD_CDict* ZSTD_createCDict_advanced2(
(int)dictLoadMethod, (int)dictContentType);
}
/* The implementation above intentionally keeps the private workspace and
* object operations in C. Rust only sequences those operations and handles
* the public construction policy. */
static void* ZSTD_rust_createCDict_create(
void* context, const void* dict, size_t dictSize,
int dictLoadMethod, int dictContentType,
@@ -6936,35 +6996,56 @@ typedef struct {
ZSTD_dictContentType_e dictContentType;
ZSTD_compressionParameters cParams;
ZSTD_ParamSwitch_e useRowMatchFinder;
ZSTD_cwksp workspaceState;
ZSTD_CCtx_params params;
} ZSTD_rust_initStaticCDictContext;
static void* ZSTD_rust_initStaticCDict_init(void* opaque)
static void ZSTD_rust_initStaticCDict_createWorkspace(void* opaque)
{
ZSTD_rust_initStaticCDictContext* const context =
(ZSTD_rust_initStaticCDictContext*)opaque;
ZSTD_cwksp ws;
ZSTD_CDict* cdict;
ZSTD_CCtx_params params;
ZSTD_cwksp_init(&ws, context->workspace, context->workspaceSize,
ZSTD_cwksp_init(&context->workspaceState, context->workspace, context->workspaceSize,
ZSTD_cwksp_static_alloc);
cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict));
if (cdict == NULL) return NULL;
ZSTD_cwksp_move(&cdict->workspace, &ws);
}
ZSTD_CCtxParams_init(&params, 0);
params.cParams = context->cParams;
params.useRowMatchFinder = context->useRowMatchFinder;
cdict->useRowMatchFinder = context->useRowMatchFinder;
cdict->compressionLevel = ZSTD_NO_CLEVEL;
static void* ZSTD_rust_initStaticCDict_reserveObject(void* opaque)
{
ZSTD_rust_initStaticCDictContext* const context =
(ZSTD_rust_initStaticCDictContext*)opaque;
return ZSTD_cwksp_reserve_object(&context->workspaceState, sizeof(ZSTD_CDict));
}
if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
context->dict, context->dictSize,
context->dictLoadMethod,
context->dictContentType,
params) ))
static void ZSTD_rust_initStaticCDict_moveWorkspace(
void* opaque, void* cdict)
{
ZSTD_rust_initStaticCDictContext* const context =
(ZSTD_rust_initStaticCDictContext*)opaque;
ZSTD_cwksp_move(&((ZSTD_CDict*)cdict)->workspace, &context->workspaceState);
}
static void ZSTD_rust_initStaticCDict_initialize(void* opaque, void* cdict)
{
ZSTD_rust_initStaticCDictContext* const context =
(ZSTD_rust_initStaticCDictContext*)opaque;
ZSTD_CDict* const dictionary = (ZSTD_CDict*)cdict;
ZSTD_CCtx_params* const params = &context->params;
ZSTD_CCtxParams_init(params, 0);
params->cParams = context->cParams;
params->useRowMatchFinder = context->useRowMatchFinder;
dictionary->useRowMatchFinder = context->useRowMatchFinder;
dictionary->compressionLevel = ZSTD_NO_CLEVEL;
}
static void* ZSTD_rust_initStaticCDict_init(void* opaque, void* cdict)
{
ZSTD_rust_initStaticCDictContext const* const context =
(const ZSTD_rust_initStaticCDictContext*)opaque;
if (ZSTD_isError( ZSTD_initCDict_internal(
(ZSTD_CDict*)cdict, context->dict, context->dictSize,
context->dictLoadMethod, context->dictContentType,
context->params) ))
return NULL;
return cdict;
}
@@ -7005,6 +7086,10 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
state.workspace = workspace;
state.workspaceSize = workspaceSize;
state.neededSize = neededSize;
state.createWorkspace = ZSTD_rust_initStaticCDict_createWorkspace;
state.reserveObject = ZSTD_rust_initStaticCDict_reserveObject;
state.moveWorkspace = ZSTD_rust_initStaticCDict_moveWorkspace;
state.initialize = ZSTD_rust_initStaticCDict_initialize;
state.init = ZSTD_rust_initStaticCDict_init;
return (const ZSTD_CDict*)ZSTD_rust_initStaticCDict(&state);
}