ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(compress): move simple-path eligibility to Rust

Browse Source 
Project the private requested-parameter and context state into a fixed scalar ABI record, leaving only C's parameter-selection leaf and private layout access in the shim. Rust now owns the complete-input eligibility predicate, including advanced-state exclusions, LDM sentinel handling, and the final fast/double-fast strategy gate, while preserving the requested-level and INT_MIN results.

Add focused policy tests for fast and double-fast acceptance, representative advanced-state rejection, the accepted LDM sentinel, non-fast strategies, and null state.

Test Plan:

- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo +nightly fmt --manifest-path rust/Cargo.toml --all -- --check

- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings

- ulimit -v 41943040; CARGO_BUILD_JOBS=1 make -j1

- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings

- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/cli/Cargo.toml --all-targets

- ulimit -v 41943040; CARGO_BUILD_JOBS=1 make -j1 -C tests test
This commit is contained in:
ddidderr
2026-07-21 12:36:26 +02:00
parent 0735a5fd7f
commit 5c6302a3c3
2 changed files with 304 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+118 -68
View File
@@ -1043,6 +1043,64 @@ typedef char ZSTD_rust_compress_stream2_mt_coordinator_state_layout[
== 6 * sizeof(void*))
? 1 : -1];
int ZSTD_rust_simpleCompress2Level(const void* cctx, size_t srcSize);
typedef struct {
int format;
int contentSizeFlag;
int checksumFlag;
int cParamsWindowLog;
int cParamsChainLog;
int cParamsHashLog;
int cParamsSearchLog;
int cParamsMinMatch;
int cParamsTargetLength;
int requestedStrategy;
int forceWindow;
int targetCBlockSizeSet;
int srcSizeHintSet;
int attachDictPref;
int literalCompressionMode;
int nbWorkersSet;
int jobSizeSet;
int overlapLogSet;
int rsyncable;
int ldmEnable;
int ldmHashLogSet;
int ldmBucketSizeLog;
int ldmMinMatchLengthSet;
int ldmHashRateLog;
int ldmWindowLogSet;
int dedicatedDictSearch;
int inBufferMode;
int outBufferMode;
int blockDelimiters;
int validateSequences;
int postBlockSplitter;
int preBlockSplitterLevelSet;
int maxBlockSizeSet;
int maxBlockSizeSetByApi;
int useRowMatchFinder;
int deterministicRefPrefix;
int prefetchCDictTables;
int matchFinderFallback;
int extSeqProdFuncSet;
int searchForExternalRepcodes;
int poolSet;
int collectSequences;
int cdictSet;
int prefixDictSet;
int localDictSet;
int compressionLevel;
int selectedStrategy;
} ZSTD_rust_simpleCompress2LevelState;
int ZSTD_rust_simpleCompress2LevelPolicy(
const ZSTD_rust_simpleCompress2LevelState* state);
typedef char ZSTD_rust_simple_compress2_level_state_layout[
(offsetof(ZSTD_rust_simpleCompress2LevelState, format) == 0
&& offsetof(ZSTD_rust_simpleCompress2LevelState, selectedStrategy)
== 46 * sizeof(int)
&& sizeof(ZSTD_rust_simpleCompress2LevelState)
== 47 * sizeof(int))
? 1 : -1];
typedef int (*ZSTD_rust_simpleCompress2Level_f)(const void* cctx, size_t srcSize);
typedef struct {
int streamStage;
@@ -7236,78 +7294,70 @@ void ZSTD_rust_markSimpleCompression2Complete(void* cctx)
((ZSTD_CCtx*)cctx)->rustSimpleCompress2Completed = 1;
}
/* Return the requested level when the context has only the parameters that
* the Rust frame path currently implements. All other contexts continue
* through ZSTD_compress2_c(), preserving the full C stateful implementation
* while this boundary is migrated incrementally. */
/* Project private context state before Rust evaluates the simple-path policy.
* The parameter-selection leaf remains C-owned because it depends on the
* private compression-parameter tables and implementation details. */
int ZSTD_rust_simpleCompress2Level(const void* opaqueCctx, size_t srcSize)
{
ZSTD_CCtx const* const cctx = (ZSTD_CCtx const*)opaqueCctx;
ZSTD_CCtx_params const* const params = cctx ? &cctx->requestedParams : NULL;
ZSTD_compressionParameters const cParams = params
? ZSTD_getCParams_internal(params->compressionLevel,
srcSize, 0,
ZSTD_cpm_noAttachDict)
: (ZSTD_compressionParameters){ 0 };
if (params == NULL
|| params->format != ZSTD_f_zstd1
|| params->fParams.contentSizeFlag == 0
|| params->fParams.checksumFlag != 0
|| params->cParams.windowLog != 0
|| params->cParams.chainLog != 0
|| params->cParams.hashLog != 0
|| params->cParams.searchLog != 0
|| params->cParams.minMatch != 0
|| params->cParams.targetLength != 0
|| params->cParams.strategy != 0
|| params->forceWindow != 0
|| params->targetCBlockSize != 0
|| params->srcSizeHint != 0
|| params->attachDictPref != ZSTD_dictDefaultAttach
|| params->literalCompressionMode != (ZSTD_ParamSwitch_e)ZSTD_lcm_auto
|| params->nbWorkers != 0
|| params->jobSize != 0
|| params->overlapLog != 0
|| params->rsyncable != 0
|| params->ldmParams.enableLdm != ZSTD_ps_auto
|| params->ldmParams.hashLog != 0
|| (params->ldmParams.bucketSizeLog != 0
&& params->ldmParams.bucketSizeLog != 9999)
|| params->ldmParams.minMatchLength != 0
|| (params->ldmParams.hashRateLog != 0
&& params->ldmParams.hashRateLog != 9999)
|| params->ldmParams.windowLog != 0
|| params->enableDedicatedDictSearch != 0
|| params->inBufferMode != ZSTD_bm_buffered
|| params->outBufferMode != ZSTD_bm_buffered
|| params->blockDelimiters != ZSTD_sf_noBlockDelimiters
|| params->validateSequences != 0
|| params->postBlockSplitter != ZSTD_ps_auto
|| params->preBlockSplitter_level != 0
|| params->maxBlockSize != 0
|| cctx->rustSimpleCompress2MaxBlockSizeSet != 0
|| params->useRowMatchFinder != ZSTD_ps_auto
|| params->deterministicRefPrefix != 0
|| params->prefetchCDictTables != ZSTD_ps_auto
|| params->enableMatchFinderFallback != 0
|| params->extSeqProdFunc != NULL
|| params->searchForExternalRepcodes != ZSTD_ps_auto
|| cctx->pool != NULL
|| cctx->seqCollector.collectSequences != 0
|| cctx->cdict != NULL
|| cctx->prefixDict.dict != NULL
|| cctx->localDict.dict != NULL
|| cctx->localDict.dictBuffer != NULL
|| cctx->localDict.cdict != NULL) {
return (-2147483647 - 1);
if (cctx == NULL) return (-2147483647 - 1);
{
ZSTD_CCtx_params const* const params = &cctx->requestedParams;
ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(
params->compressionLevel, srcSize, 0, ZSTD_cpm_noAttachDict);
ZSTD_rust_simpleCompress2LevelState const state = {
(int)params->format,
(int)params->fParams.contentSizeFlag,
(int)params->fParams.checksumFlag,
(int)params->cParams.windowLog,
(int)params->cParams.chainLog,
(int)params->cParams.hashLog,
(int)params->cParams.searchLog,
(int)params->cParams.minMatch,
(int)params->cParams.targetLength,
(int)params->cParams.strategy,
params->forceWindow != 0,
(int)(params->targetCBlockSize != 0),
(int)(params->srcSizeHint != 0),
(int)params->attachDictPref,
(int)params->literalCompressionMode,
(int)(params->nbWorkers != 0),
(int)(params->jobSize != 0),
(int)(params->overlapLog != 0),
params->rsyncable != 0,
(int)params->ldmParams.enableLdm,
(int)(params->ldmParams.hashLog != 0),
(int)params->ldmParams.bucketSizeLog,
(int)(params->ldmParams.minMatchLength != 0),
(int)params->ldmParams.hashRateLog,
(int)(params->ldmParams.windowLog != 0),
params->enableDedicatedDictSearch != 0,
(int)params->inBufferMode,
(int)params->outBufferMode,
(int)params->blockDelimiters,
params->validateSequences != 0,
(int)params->postBlockSplitter,
(int)(params->preBlockSplitter_level != 0),
(int)(params->maxBlockSize != 0),
(int)(cctx->rustSimpleCompress2MaxBlockSizeSet != 0),
(int)params->useRowMatchFinder,
params->deterministicRefPrefix != 0,
(int)params->prefetchCDictTables,
params->enableMatchFinderFallback != 0,
(int)(params->extSeqProdFunc != NULL),
(int)params->searchForExternalRepcodes,
(int)(cctx->pool != NULL),
cctx->seqCollector.collectSequences != 0,
(int)(cctx->cdict != NULL),
(int)(cctx->prefixDict.dict != NULL),
(int)(cctx->localDict.dict != NULL
|| cctx->localDict.dictBuffer != NULL
|| cctx->localDict.cdict != NULL),
params->compressionLevel,
(int)cParams.strategy
};
return ZSTD_rust_simpleCompress2LevelPolicy(&state);
}
/* The Rust frame leaf currently implements only the fast and double-fast
* match finders. Keep lazy and optimal strategies on the stateful path,
* which owns their strategy-specific match-table lifecycle. */
if (cParams.strategy != ZSTD_fast && cParams.strategy != ZSTD_dfast) {
return (-2147483647 - 1);
}
return params->compressionLevel;
}
/* Return the simple level only while a new stream frame can be initialized.
rust/src/zstd_compress.rs
+186
View File
@@ -151,6 +151,143 @@ const _: () = {
pub type ZSTD_rust_simpleCompress2LevelFn = unsafe extern "C" fn(*const c_void, usize) -> c_int;
const ZSTD_DICT_DEFAULT_ATTACH: c_int = 0;
const ZSTD_LCM_AUTO: c_int = 0;
const ZSTD_LDM_UNUSED_LOG: c_int = 9999;
/// Scalar projection of the private context and requested parameters used to
/// select the complete-input simple-compression fast path.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTD_rust_simpleCompress2LevelState {
format: c_int,
content_size_flag: c_int,
checksum_flag: c_int,
cparams_window_log: c_int,
cparams_chain_log: c_int,
cparams_hash_log: c_int,
cparams_search_log: c_int,
cparams_min_match: c_int,
cparams_target_length: c_int,
requested_strategy: c_int,
force_window: c_int,
target_c_block_size_set: c_int,
src_size_hint_set: c_int,
attach_dict_pref: c_int,
literal_compression_mode: c_int,
nb_workers_set: c_int,
job_size_set: c_int,
overlap_log_set: c_int,
rsyncable: c_int,
ldm_enable: c_int,
ldm_hash_log_set: c_int,
ldm_bucket_size_log: c_int,
ldm_min_match_length_set: c_int,
ldm_hash_rate_log: c_int,
ldm_window_log_set: c_int,
dedicated_dict_search: c_int,
in_buffer_mode: c_int,
out_buffer_mode: c_int,
block_delimiters: c_int,
validate_sequences: c_int,
post_block_splitter: c_int,
pre_block_splitter_level_set: c_int,
max_block_size_set: c_int,
max_block_size_set_by_api: c_int,
use_row_match_finder: c_int,
deterministic_ref_prefix: c_int,
prefetch_cdict_tables: c_int,
match_finder_fallback: c_int,
ext_seq_prod_func_set: c_int,
search_for_external_repcodes: c_int,
pool_set: c_int,
collect_sequences: c_int,
cdict_set: c_int,
prefix_dict_set: c_int,
local_dict_set: c_int,
compression_level: c_int,
selected_strategy: c_int,
}
const _: () = {
assert!(offset_of!(ZSTD_rust_simpleCompress2LevelState, format) == 0);
assert!(
offset_of!(ZSTD_rust_simpleCompress2LevelState, selected_strategy)
== 46 * size_of::<c_int>()
);
assert!(size_of::<ZSTD_rust_simpleCompress2LevelState>() == 47 * size_of::<c_int>());
};
#[inline]
fn simple_compress2_level_policy(state: &ZSTD_rust_simpleCompress2LevelState) -> c_int {
if state.format != ZSTD_F_ZSTD1
|| state.content_size_flag == 0
|| state.checksum_flag != 0
|| state.cparams_window_log != 0
|| state.cparams_chain_log != 0
|| state.cparams_hash_log != 0
|| state.cparams_search_log != 0
|| state.cparams_min_match != 0
|| state.cparams_target_length != 0
|| state.requested_strategy != 0
|| state.force_window != 0
|| state.target_c_block_size_set != 0
|| state.src_size_hint_set != 0
|| state.attach_dict_pref != ZSTD_DICT_DEFAULT_ATTACH
|| state.literal_compression_mode != ZSTD_LCM_AUTO
|| state.nb_workers_set != 0
|| state.job_size_set != 0
|| state.overlap_log_set != 0
|| state.rsyncable != 0
|| state.ldm_enable != ZSTD_RUST_PS_AUTO
|| state.ldm_hash_log_set != 0
|| (state.ldm_bucket_size_log != 0 && state.ldm_bucket_size_log != ZSTD_LDM_UNUSED_LOG)
|| state.ldm_min_match_length_set != 0
|| (state.ldm_hash_rate_log != 0 && state.ldm_hash_rate_log != ZSTD_LDM_UNUSED_LOG)
|| state.ldm_window_log_set != 0
|| state.dedicated_dict_search != 0
|| state.in_buffer_mode != ZSTD_BM_BUFFERED
|| state.out_buffer_mode != ZSTD_BM_BUFFERED
|| state.block_delimiters != ZSTD_SF_NO_BLOCK_DELIMITERS
|| state.validate_sequences != 0
|| state.post_block_splitter != ZSTD_RUST_PS_AUTO
|| state.pre_block_splitter_level_set != 0
|| state.max_block_size_set != 0
|| state.max_block_size_set_by_api != 0
|| state.use_row_match_finder != ZSTD_RUST_PS_AUTO
|| state.deterministic_ref_prefix != 0
|| state.prefetch_cdict_tables != ZSTD_RUST_PS_AUTO
|| state.match_finder_fallback != 0
|| state.ext_seq_prod_func_set != 0
|| state.search_for_external_repcodes != ZSTD_RUST_PS_AUTO
|| state.pool_set != 0
|| state.collect_sequences != 0
|| state.cdict_set != 0
|| state.prefix_dict_set != 0
|| state.local_dict_set != 0
{
return c_int::MIN;
}
if state.selected_strategy != ZSTD_FAST && state.selected_strategy != ZSTD_DFAST {
return c_int::MIN;
}
state.compression_level
}
/// Decide whether a projected context can use the complete-input Rust frame
/// compressor. C retains private-layout projection and parameter selection;
/// Rust owns the eligibility predicate and strategy gate.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_simpleCompress2LevelPolicy(
state: *const ZSTD_rust_simpleCompress2LevelState,
) -> c_int {
let Some(state) = (unsafe { state.as_ref() }) else {
return c_int::MIN;
};
simple_compress2_level_policy(state)
}
const ZSTD_WINDOW_START_INDEX: u32 = 2;
const ZSTD_DUBT_UNSORTED_MARK: u32 = 1;
const ZSTD_INDEXOVERFLOW_MARGIN: usize = 16usize << 20;
@@ -14834,6 +14971,55 @@ mod tests {
);
}
fn simple_compress2_level_policy_state() -> ZSTD_rust_simpleCompress2LevelState {
ZSTD_rust_simpleCompress2LevelState {
format: ZSTD_F_ZSTD1,
content_size_flag: 1,
compression_level: 5,
selected_strategy: ZSTD_FAST,
..Default::default()
}
}
#[test]
fn simple_compress2_level_policy_accepts_fast_and_double_fast() {
for strategy in [ZSTD_FAST, ZSTD_DFAST] {
let mut state = simple_compress2_level_policy_state();
state.selected_strategy = strategy;
assert_eq!(simple_compress2_level_policy(&state), 5);
}
}
#[test]
fn simple_compress2_level_policy_rejects_advanced_state() {
let mut state = simple_compress2_level_policy_state();
state.checksum_flag = 1;
assert_eq!(simple_compress2_level_policy(&state), c_int::MIN);
let mut state = simple_compress2_level_policy_state();
state.cdict_set = 1;
assert_eq!(simple_compress2_level_policy(&state), c_int::MIN);
let mut state = simple_compress2_level_policy_state();
state.ldm_bucket_size_log = 1;
assert_eq!(simple_compress2_level_policy(&state), c_int::MIN);
let mut state = simple_compress2_level_policy_state();
state.ldm_bucket_size_log = ZSTD_LDM_UNUSED_LOG;
assert_eq!(simple_compress2_level_policy(&state), 5);
}
#[test]
fn simple_compress2_level_policy_rejects_non_fast_or_null_state() {
let mut state = simple_compress2_level_policy_state();
state.selected_strategy = ZSTD_BTLAZY2;
assert_eq!(simple_compress2_level_policy(&state), c_int::MIN);
assert_eq!(
unsafe { ZSTD_rust_simpleCompress2LevelPolicy(ptr::null()) },
c_int::MIN
);
}
#[derive(Default)]
struct SimpleStreamPolicyTestContext {
calls: usize,