ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move CCtx parameter bounds to Rust

Browse Source 
Route public CCtx parameter bounds through the Rust policy implementation while keeping build-specific MT limits supplied by the C configuration. Preserve clamping and validation behavior for migrated parameters and add focused Rust coverage.

Test Plan: cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression; cargo clippy --manifest-path rust/Cargo.toml; cargo clippy --manifest-path rust/Cargo.toml --benches; cargo clippy --manifest-path rust/Cargo.toml --tests; make -B -C lib -j2 lib; make -B -C tests -j2 test-cli-tests; make -B -C tests -j2 test-zstream
This commit is contained in:
ddidderr
2026-07-18 03:38:56 +02:00
parent 65bfc6bb2b
commit 6aaabce208
2 changed files with 259 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+24 -212
View File
@@ -81,12 +81,15 @@ size_t ZSTD_rust_params_maxNbSeq(size_t blockSize, U32 minMatch,
size_t ZSTD_rust_params_resolveMaxBlockSize(size_t maxBlockSize);
/* CCtx parameter state is mirrored by rust/src/zstd_compress_params_api.rs.
* Keep bounds and build-configuration policy in C, and expose only the
* private helpers required by that Rust ABI implementation. */
* Rust owns parameter bounds and clamping; C exposes only the
* build-configuration values required by that narrow ABI. */
ZSTD_bounds ZSTD_rust_cctx_params_get_bounds(int param);
size_t ZSTD_rust_cctx_params_clamp_bounds(int param, int* value);
int ZSTD_rust_cctx_params_within_bounds(int param, int value);
int ZSTD_rust_cctx_params_is_multithreaded(void);
int ZSTD_rust_cctx_params_nb_workers_max(void);
int ZSTD_rust_cctx_params_job_size_min(void);
int ZSTD_rust_cctx_params_job_size_max(void);
ZSTD_CCtx_params* ZSTD_rust_createCCtxParams(ZSTD_customMem customMem);
size_t ZSTD_rust_freeCCtxParams(ZSTD_CCtx_params* params);
size_t ZSTD_rust_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams,
@@ -600,216 +603,7 @@ static void ZSTD_CCtxParams_setZstdParams(
ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
{
ZSTD_bounds bounds = { 0, 0, 0 };
switch(param)
{
/* The compression level and the seven core compression parameters are
* bounded by the Rust leaf. */
case ZSTD_c_compressionLevel:
case ZSTD_c_windowLog:
case ZSTD_c_hashLog:
case ZSTD_c_chainLog:
case ZSTD_c_searchLog:
case ZSTD_c_minMatch:
case ZSTD_c_targetLength:
case ZSTD_c_strategy:
return ZSTD_rust_params_getBounds((int)param);
case ZSTD_c_contentSizeFlag:
bounds.lowerBound = 0;
bounds.upperBound = 1;
return bounds;
case ZSTD_c_checksumFlag:
bounds.lowerBound = 0;
bounds.upperBound = 1;
return bounds;
case ZSTD_c_dictIDFlag:
bounds.lowerBound = 0;
bounds.upperBound = 1;
return bounds;
case ZSTD_c_nbWorkers:
bounds.lowerBound = 0;
#ifdef ZSTD_MULTITHREAD
bounds.upperBound = ZSTDMT_NBWORKERS_MAX;
#else
bounds.upperBound = 0;
#endif
return bounds;
case ZSTD_c_jobSize:
bounds.lowerBound = 0;
#ifdef ZSTD_MULTITHREAD
bounds.upperBound = ZSTDMT_JOBSIZE_MAX;
#else
bounds.upperBound = 0;
#endif
return bounds;
case ZSTD_c_overlapLog:
#ifdef ZSTD_MULTITHREAD
bounds.lowerBound = ZSTD_OVERLAPLOG_MIN;
bounds.upperBound = ZSTD_OVERLAPLOG_MAX;
#else
bounds.lowerBound = 0;
bounds.upperBound = 0;
#endif
return bounds;
case ZSTD_c_enableDedicatedDictSearch:
bounds.lowerBound = 0;
bounds.upperBound = 1;
return bounds;
case ZSTD_c_enableLongDistanceMatching:
bounds.lowerBound = (int)ZSTD_ps_auto;
bounds.upperBound = (int)ZSTD_ps_disable;
return bounds;
case ZSTD_c_ldmHashLog:
bounds.lowerBound = ZSTD_LDM_HASHLOG_MIN;
bounds.upperBound = ZSTD_LDM_HASHLOG_MAX;
return bounds;
case ZSTD_c_ldmMinMatch:
bounds.lowerBound = ZSTD_LDM_MINMATCH_MIN;
bounds.upperBound = ZSTD_LDM_MINMATCH_MAX;
return bounds;
case ZSTD_c_ldmBucketSizeLog:
bounds.lowerBound = ZSTD_LDM_BUCKETSIZELOG_MIN;
bounds.upperBound = ZSTD_LDM_BUCKETSIZELOG_MAX;
return bounds;
case ZSTD_c_ldmHashRateLog:
bounds.lowerBound = ZSTD_LDM_HASHRATELOG_MIN;
bounds.upperBound = ZSTD_LDM_HASHRATELOG_MAX;
return bounds;
/* experimental parameters */
case ZSTD_c_rsyncable:
bounds.lowerBound = 0;
bounds.upperBound = 1;
return bounds;
case ZSTD_c_forceMaxWindow :
bounds.lowerBound = 0;
bounds.upperBound = 1;
return bounds;
case ZSTD_c_format:
ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless);
bounds.lowerBound = ZSTD_f_zstd1;
bounds.upperBound = ZSTD_f_zstd1_magicless; /* note : how to ensure at compile time that this is the highest value enum ? */
return bounds;
case ZSTD_c_forceAttachDict:
ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceLoad);
bounds.lowerBound = ZSTD_dictDefaultAttach;
bounds.upperBound = ZSTD_dictForceLoad; /* note : how to ensure at compile time that this is the highest value enum ? */
return bounds;
case ZSTD_c_literalCompressionMode:
ZSTD_STATIC_ASSERT(ZSTD_ps_auto < ZSTD_ps_enable && ZSTD_ps_enable < ZSTD_ps_disable);
bounds.lowerBound = (int)ZSTD_ps_auto;
bounds.upperBound = (int)ZSTD_ps_disable;
return bounds;
case ZSTD_c_targetCBlockSize:
bounds.lowerBound = ZSTD_TARGETCBLOCKSIZE_MIN;
bounds.upperBound = ZSTD_TARGETCBLOCKSIZE_MAX;
return bounds;
case ZSTD_c_srcSizeHint:
bounds.lowerBound = ZSTD_SRCSIZEHINT_MIN;
bounds.upperBound = ZSTD_SRCSIZEHINT_MAX;
return bounds;
case ZSTD_c_stableInBuffer:
case ZSTD_c_stableOutBuffer:
bounds.lowerBound = (int)ZSTD_bm_buffered;
bounds.upperBound = (int)ZSTD_bm_stable;
return bounds;
case ZSTD_c_blockDelimiters:
bounds.lowerBound = (int)ZSTD_sf_noBlockDelimiters;
bounds.upperBound = (int)ZSTD_sf_explicitBlockDelimiters;
return bounds;
case ZSTD_c_validateSequences:
bounds.lowerBound = 0;
bounds.upperBound = 1;
return bounds;
case ZSTD_c_splitAfterSequences:
bounds.lowerBound = (int)ZSTD_ps_auto;
bounds.upperBound = (int)ZSTD_ps_disable;
return bounds;
case ZSTD_c_blockSplitterLevel:
bounds.lowerBound = 0;
bounds.upperBound = ZSTD_BLOCKSPLITTER_LEVEL_MAX;
return bounds;
case ZSTD_c_useRowMatchFinder:
bounds.lowerBound = (int)ZSTD_ps_auto;
bounds.upperBound = (int)ZSTD_ps_disable;
return bounds;
case ZSTD_c_deterministicRefPrefix:
bounds.lowerBound = 0;
bounds.upperBound = 1;
return bounds;
case ZSTD_c_prefetchCDictTables:
bounds.lowerBound = (int)ZSTD_ps_auto;
bounds.upperBound = (int)ZSTD_ps_disable;
return bounds;
case ZSTD_c_enableSeqProducerFallback:
bounds.lowerBound = 0;
bounds.upperBound = 1;
return bounds;
case ZSTD_c_maxBlockSize:
bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN;
bounds.upperBound = ZSTD_BLOCKSIZE_MAX;
return bounds;
case ZSTD_c_repcodeResolution:
bounds.lowerBound = (int)ZSTD_ps_auto;
bounds.upperBound = (int)ZSTD_ps_disable;
return bounds;
default:
bounds.error = ERROR(parameter_unsupported);
return bounds;
}
}
/* ZSTD_cParam_clampBounds:
* Clamps the value into the bounded range.
*/
static size_t ZSTD_cParam_clampBounds(ZSTD_cParameter cParam, int* value)
{
ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam);
if (ZSTD_isError(bounds.error)) return bounds.error;
if (*value < bounds.lowerBound) *value = bounds.lowerBound;
if (*value > bounds.upperBound) *value = bounds.upperBound;
return 0;
}
size_t ZSTD_rust_cctx_params_clamp_bounds(int param, int* value)
{
return ZSTD_cParam_clampBounds((ZSTD_cParameter)param, value);
}
int ZSTD_rust_cctx_params_within_bounds(int param, int value)
{
return ZSTD_cParam_withinBounds((ZSTD_cParameter)param, value);
return ZSTD_rust_cctx_params_get_bounds((int)param);
}
int ZSTD_rust_cctx_params_is_multithreaded(void)
@@ -821,6 +615,15 @@ int ZSTD_rust_cctx_params_is_multithreaded(void)
#endif
}
int ZSTD_rust_cctx_params_nb_workers_max(void)
{
#ifdef ZSTD_MULTITHREAD
return ZSTDMT_NBWORKERS_MAX;
#else
return 0;
#endif
}
int ZSTD_rust_cctx_params_job_size_min(void)
{
#ifdef ZSTD_MULTITHREAD
@@ -830,6 +633,15 @@ int ZSTD_rust_cctx_params_job_size_min(void)
#endif
}
int ZSTD_rust_cctx_params_job_size_max(void)
{
#ifdef ZSTD_MULTITHREAD
return ZSTDMT_JOBSIZE_MAX;
#else
return 0;
#endif
}
#define BOUNDCHECK(cParam, val) \
do { \
RETURN_ERROR_IF(!ZSTD_cParam_withinBounds(cParam,val), \
rust/src/zstd_compress_params_api.rs
+235 -39
View File
@@ -12,12 +12,10 @@
//! default-resolution policy.
use crate::errors::{ZstdErrorCode, ERROR};
#[cfg(test)]
use crate::zstd_compress_params::ZSTD_bounds;
use crate::zstd_compress_params::{
ZSTD_compressionParameters, ZSTD_frameParameters, ZSTD_parameters,
ZSTD_rust_params_checkCParams, ZSTD_rust_params_resolveMaxBlockSize, ZSTD_RUST_PS_DISABLE,
ZSTD_RUST_PS_ENABLE,
ZSTD_bounds, ZSTD_compressionParameters, ZSTD_frameParameters, ZSTD_parameters,
ZSTD_rust_params_checkCParams, ZSTD_rust_params_getBounds,
ZSTD_rust_params_resolveMaxBlockSize, ZSTD_RUST_PS_DISABLE, ZSTD_RUST_PS_ENABLE,
};
use std::mem::size_of;
use std::os::raw::{c_int, c_void};
@@ -125,9 +123,7 @@ pub struct ZSTD_CCtx_params {
const DEFAULT_CLEVEL: c_int = 3;
const NO_CLEVEL: c_int = 0;
#[cfg(test)]
const PS_AUTO: c_int = 0;
#[cfg(test)]
const PS_DISABLE: c_int = 2;
/* Private strategy values used by the existing C policy helpers. */
@@ -135,9 +131,7 @@ const STRATEGY_GREEDY: c_int = 3;
const STRATEGY_LAZY2: c_int = 5;
const STRATEGY_BTOPT: c_int = 7;
#[cfg(test)]
const BLOCKSIZE_MAX: usize = 1 << 17;
#[cfg(test)]
const BLOCKSIZE_MAX_MIN: c_int = 1 << 10;
const TARGET_C_BLOCK_SIZE_MIN: c_int = 1340;
@@ -184,34 +178,85 @@ const C_BLOCK_SPLITTER_LEVEL: c_int = 1017;
#[cfg(not(test))]
unsafe extern "C" {
fn ZSTD_rust_cctx_params_clamp_bounds(param: c_int, value: *mut c_int) -> usize;
fn ZSTD_rust_cctx_params_within_bounds(param: c_int, value: c_int) -> c_int;
fn ZSTD_rust_cctx_params_is_multithreaded() -> c_int;
fn ZSTD_rust_cctx_params_nb_workers_max() -> c_int;
fn ZSTD_rust_cctx_params_job_size_min() -> c_int;
fn ZSTD_rust_cctx_params_job_size_max() -> c_int;
}
#[derive(Clone, Copy)]
struct BoundsConfig {
multithreaded: bool,
nb_workers_max: c_int,
job_size_max: c_int,
}
#[cfg(not(test))]
unsafe fn current_bounds_config() -> BoundsConfig {
BoundsConfig {
multithreaded: unsafe { ZSTD_rust_cctx_params_is_multithreaded() != 0 },
nb_workers_max: unsafe { ZSTD_rust_cctx_params_nb_workers_max() },
job_size_max: unsafe { ZSTD_rust_cctx_params_job_size_max() },
}
}
#[cfg(test)]
fn fallback_bounds(param: c_int) -> ZSTD_bounds {
unsafe fn current_bounds_config() -> BoundsConfig {
BoundsConfig {
multithreaded: false,
nb_workers_max: 0,
job_size_max: 0,
}
}
fn bounds_for(param: c_int, config: BoundsConfig) -> ZSTD_bounds {
let (lower, upper) = match param {
C_COMPRESSION_LEVEL | C_WINDOW_LOG | C_HASH_LOG | C_CHAIN_LOG | C_SEARCH_LOG
| C_MIN_MATCH | C_TARGET_LENGTH | C_STRATEGY => {
return crate::zstd_compress_params::ZSTD_rust_params_getBounds(param)
}
| C_MIN_MATCH | C_TARGET_LENGTH | C_STRATEGY => return ZSTD_rust_params_getBounds(param),
C_FORMAT => (0, 1),
C_FORCE_ATTACH_DICT => (0, 3),
C_LITERAL_COMPRESSION_MODE => (0, 2),
C_CONTENT_SIZE_FLAG | C_CHECKSUM_FLAG | C_DICT_ID_FLAG => (0, 1),
C_NB_WORKERS | C_JOB_SIZE | C_OVERLAP_LOG => (0, 0),
C_NB_WORKERS => (
0,
if config.multithreaded {
config.nb_workers_max
} else {
0
},
),
C_JOB_SIZE => (
0,
if config.multithreaded {
config.job_size_max
} else {
0
},
),
C_OVERLAP_LOG => {
if config.multithreaded {
(0, 9)
} else {
(0, 0)
}
}
C_ENABLE_DEDICATED_DICT_SEARCH => (0, 1),
C_ENABLE_LDM
| C_SPLIT_AFTER_SEQUENCES
| C_USE_ROW_MATCH_FINDER
| C_PREFETCH_CDICT_TABLES
| C_REPCODE_RESOLUTION => (PS_AUTO, PS_DISABLE),
C_LDM_HASH_LOG => (6, 30),
C_LDM_HASH_LOG => {
let bounds = ZSTD_rust_params_getBounds(C_HASH_LOG);
(bounds.lowerBound, bounds.upperBound)
}
C_LDM_MIN_MATCH => (4, 4096),
C_LDM_BUCKET_SIZE_LOG => (1, 8),
C_LDM_HASH_RATE_LOG => (0, if size_of::<usize>() == 4 { 24 } else { 25 }),
C_LDM_HASH_RATE_LOG => {
let window_log = ZSTD_rust_params_getBounds(C_WINDOW_LOG);
let hash_log = ZSTD_rust_params_getBounds(C_HASH_LOG);
(0, window_log.upperBound - hash_log.lowerBound)
}
C_RSYNCABLE
| C_FORCE_MAX_WINDOW
| C_VALIDATE_SEQUENCES
@@ -237,34 +282,53 @@ fn fallback_bounds(param: c_int) -> ZSTD_bounds {
}
}
#[inline]
unsafe fn current_bounds(param: c_int) -> ZSTD_bounds {
bounds_for(param, unsafe { current_bounds_config() })
}
#[inline]
unsafe fn within_bounds(param: c_int, value: c_int) -> bool {
#[cfg(not(test))]
{
unsafe { ZSTD_rust_cctx_params_within_bounds(param, value) != 0 }
}
#[cfg(test)]
{
let bounds = fallback_bounds(param);
bounds.error == 0 && value >= bounds.lowerBound && value <= bounds.upperBound
within_bounds_for(param, value, unsafe { current_bounds_config() })
}
#[inline]
fn within_bounds_for(param: c_int, value: c_int, config: BoundsConfig) -> bool {
let bounds = bounds_for(param, config);
bounds.error == 0 && value >= bounds.lowerBound && value <= bounds.upperBound
}
#[inline]
fn clamp_bounds_for(param: c_int, value: &mut c_int, config: BoundsConfig) -> usize {
let bounds = bounds_for(param, config);
if bounds.error != 0 {
return bounds.error;
}
*value = (*value).clamp(bounds.lowerBound, bounds.upperBound);
0
}
#[inline]
unsafe fn clamp_bounds(param: c_int, value: &mut c_int) -> usize {
#[cfg(not(test))]
{
unsafe { ZSTD_rust_cctx_params_clamp_bounds(param, value) }
}
#[cfg(test)]
{
let bounds = fallback_bounds(param);
if bounds.error != 0 {
return bounds.error;
}
*value = (*value).clamp(bounds.lowerBound, bounds.upperBound);
0
}
clamp_bounds_for(param, value, unsafe { current_bounds_config() })
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_cctx_params_get_bounds(param: c_int) -> ZSTD_bounds {
unsafe { current_bounds(param) }
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_cctx_params_clamp_bounds(
param: c_int,
value: *mut c_int,
) -> usize {
unsafe { clamp_bounds(param, &mut *value) }
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_cctx_params_within_bounds(param: c_int, value: c_int) -> c_int {
unsafe { within_bounds(param, value) as c_int }
}
#[inline]
@@ -1006,6 +1070,138 @@ mod tests {
);
}
#[test]
fn migrated_non_core_bounds_preserve_pointer_and_thread_configuration() {
let single_threaded = BoundsConfig {
multithreaded: false,
nb_workers_max: 0,
job_size_max: 0,
};
let expected = [
(C_FORMAT, 0, 1),
(C_FORCE_ATTACH_DICT, 0, 3),
(C_LITERAL_COMPRESSION_MODE, 0, 2),
(C_CONTENT_SIZE_FLAG, 0, 1),
(C_CHECKSUM_FLAG, 0, 1),
(C_DICT_ID_FLAG, 0, 1),
(C_NB_WORKERS, 0, 0),
(C_JOB_SIZE, 0, 0),
(C_OVERLAP_LOG, 0, 0),
(C_ENABLE_DEDICATED_DICT_SEARCH, 0, 1),
(C_ENABLE_LDM, PS_AUTO, PS_DISABLE),
(C_LDM_HASH_LOG, 6, 30),
(C_LDM_MIN_MATCH, 4, 4096),
(C_LDM_BUCKET_SIZE_LOG, 1, 8),
(
C_LDM_HASH_RATE_LOG,
0,
if size_of::<usize>() == 4 { 24 } else { 25 },
),
(C_RSYNCABLE, 0, 1),
(C_FORCE_MAX_WINDOW, 0, 1),
(C_VALIDATE_SEQUENCES, 0, 1),
(C_SPLIT_AFTER_SEQUENCES, PS_AUTO, PS_DISABLE),
(C_USE_ROW_MATCH_FINDER, PS_AUTO, PS_DISABLE),
(C_DETERMINISTIC_REF_PREFIX, 0, 1),
(C_PREFETCH_CDICT_TABLES, PS_AUTO, PS_DISABLE),
(C_ENABLE_SEQ_PRODUCER_FALLBACK, 0, 1),
(
C_TARGET_C_BLOCK_SIZE,
TARGET_C_BLOCK_SIZE_MIN,
BLOCKSIZE_MAX as c_int,
),
(C_SRC_SIZE_HINT, 0, c_int::MAX),
(C_STABLE_IN_BUFFER, 0, 1),
(C_STABLE_OUT_BUFFER, 0, 1),
(C_BLOCK_DELIMITERS, 0, 1),
(C_BLOCK_SPLITTER_LEVEL, 0, 6),
(C_MAX_BLOCK_SIZE, BLOCKSIZE_MAX_MIN, BLOCKSIZE_MAX as c_int),
(C_REPCODE_RESOLUTION, PS_AUTO, PS_DISABLE),
];
for (param, lower, upper) in expected {
let bounds = bounds_for(param, single_threaded);
assert_eq!(bounds.error, 0, "param {param} returned an error");
assert_eq!((bounds.lowerBound, bounds.upperBound), (lower, upper));
assert!(within_bounds_for(param, lower, single_threaded));
assert!(within_bounds_for(param, upper, single_threaded));
if lower > c_int::MIN {
assert!(!within_bounds_for(param, lower - 1, single_threaded));
}
if upper < c_int::MAX {
assert!(!within_bounds_for(param, upper + 1, single_threaded));
}
}
let unsupported = bounds_for(12345, single_threaded);
assert_eq!(
unsupported.error,
ERROR(ZstdErrorCode::ParameterUnsupported)
);
let multithreaded = BoundsConfig {
multithreaded: true,
nb_workers_max: 17,
job_size_max: 987_654_321,
};
assert_eq!(
(
bounds_for(C_NB_WORKERS, multithreaded).lowerBound,
bounds_for(C_NB_WORKERS, multithreaded).upperBound
),
(0, 17)
);
assert_eq!(
(
bounds_for(C_JOB_SIZE, multithreaded).lowerBound,
bounds_for(C_JOB_SIZE, multithreaded).upperBound
),
(0, 987_654_321)
);
assert_eq!(
(
bounds_for(C_OVERLAP_LOG, multithreaded).lowerBound,
bounds_for(C_OVERLAP_LOG, multithreaded).upperBound
),
(0, 9)
);
}
#[test]
fn migrated_clamp_bounds_clamps_and_reports_unsupported_parameters() {
let multithreaded = BoundsConfig {
multithreaded: true,
nb_workers_max: 17,
job_size_max: 987_654_321,
};
let mut value = -1;
assert_eq!(clamp_bounds_for(C_FORMAT, &mut value, multithreaded), 0);
assert_eq!(value, 0);
value = c_int::MAX;
assert_eq!(
clamp_bounds_for(C_TARGET_C_BLOCK_SIZE, &mut value, multithreaded),
0
);
assert_eq!(value, BLOCKSIZE_MAX as c_int);
value = c_int::MAX;
assert_eq!(clamp_bounds_for(C_NB_WORKERS, &mut value, multithreaded), 0);
assert_eq!(value, 17);
value = -1;
assert_eq!(clamp_bounds_for(C_JOB_SIZE, &mut value, multithreaded), 0);
assert_eq!(value, 0);
value = -1;
assert_eq!(
clamp_bounds_for(12345, &mut value, multithreaded),
ERROR(ZstdErrorCode::ParameterUnsupported)
);
assert_eq!(value, -1);
}
#[test]
fn advanced_init_resolves_policy_and_resets_internal_fields() {
let cases = [