feat(compress): move target block policy to Rust

Port the target-sized block action policy behind a scalar Rust ABI while keeping sequence inspection, superblock compression, block-state confirmation, diagnostics, and raw/RLE emission in C. Preserve the strict three-byte header boundary, destination-too-small fallback, and first-block RLE guard with focused tests.

Test Plan:

- cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression zstd_compress::tests::target_block

- cargo clippy --manifest-path rust/Cargo.toml

- cargo clippy --manifest-path rust/Cargo.toml --benches

- cargo clippy --manifest-path rust/Cargo.toml --tests

- cargo +nightly fmt --manifest-path rust/Cargo.toml --all

- make -B -C lib -j2 lib
This commit is contained in:
2026-07-18 06:37:07 +02:00
parent b19ca569f7
commit 21f8e8958c
2 changed files with 209 additions and 18 deletions
+32 -14
View File
@@ -66,6 +66,16 @@ void ZSTD_rust_reduceTable(U32* table, U32 size, U32 reducerValue,
U64 ZSTD_rust_advanceHashSalt(U64 hashSalt, U64 hashSaltEntropy); U64 ZSTD_rust_advanceHashSalt(U64 hashSalt, U64 hashSaltEntropy);
int ZSTD_rust_indexTooCloseToMax(size_t nextSrcBaseOffset); int ZSTD_rust_indexTooCloseToMax(size_t nextSrcBaseOffset);
int ZSTD_rust_dictTooBig(size_t loadedDictSize); int ZSTD_rust_dictTooBig(size_t loadedDictSize);
enum {
ZSTD_RUST_TARGET_CBLOCK_ACTION_RAW = 0,
ZSTD_RUST_TARGET_CBLOCK_ACTION_RLE = 1,
ZSTD_RUST_TARGET_CBLOCK_ACTION_COMPRESSED = 2,
ZSTD_RUST_TARGET_CBLOCK_ACTION_ERROR = 3
};
int ZSTD_rust_targetCBlockSizeAction(int bss, int isFirstBlock,
int maybeRLE, int isRLE,
size_t cSize, size_t srcSize,
int strategy);
/* Context-free compression-parameter selection and sizing leaves live in /* Context-free compression-parameter selection and sizing leaves live in
* Rust (rust/src/zstd_compress_params.rs). This file retains * Rust (rust/src/zstd_compress_params.rs). This file retains
@@ -3124,18 +3134,25 @@ static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc,
const void* src, size_t srcSize, const void* src, size_t srcSize,
const size_t bss, U32 lastBlock) const size_t bss, U32 lastBlock)
{ {
int const isCompress = bss == ZSTDbss_compress;
int maybeRLE = 0;
int isRLE = 0;
int action;
DEBUGLOG(6, "Attempting ZSTD_compressSuperBlock()"); DEBUGLOG(6, "Attempting ZSTD_compressSuperBlock()");
if (bss == ZSTDbss_compress) { if (isCompress) {
if (/* We don't want to emit our first block as a RLE even if it qualifies because maybeRLE = ZSTD_maybeRLE(&zc->seqStore);
* doing so will cause the decoder (cli only) to throw a "should consume all input error." isRLE = ZSTD_isRLE((BYTE const*)src, srcSize);
* This is only an issue for zstd <= v1.4.3 }
*/
!zc->isFirstBlock && action = ZSTD_rust_targetCBlockSizeAction(
ZSTD_maybeRLE(&zc->seqStore) && (int)bss, (int)zc->isFirstBlock, maybeRLE, isRLE,
ZSTD_isRLE((BYTE const*)src, srcSize)) 0, srcSize, (int)zc->appliedParams.cParams.strategy);
{ if (action == ZSTD_RUST_TARGET_CBLOCK_ACTION_RLE) {
return ZSTD_rust_rleCompressBlock(dst, dstCapacity, *(BYTE const*)src, srcSize, lastBlock); return ZSTD_rust_rleCompressBlock(dst, dstCapacity, *(BYTE const*)src, srcSize, lastBlock);
} }
if (isCompress) {
/* Attempt superblock compression. /* Attempt superblock compression.
* *
* Note that compressed size of ZSTD_compressSuperBlock() is not bound by the * Note that compressed size of ZSTD_compressSuperBlock() is not bound by the
@@ -3156,16 +3173,17 @@ static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc,
*/ */
{ size_t const cSize = { size_t const cSize =
ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock); ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock);
if (cSize != ERROR(dstSize_tooSmall)) { action = ZSTD_rust_targetCBlockSizeAction(
size_t const maxCSize = (int)bss, (int)zc->isFirstBlock, 0, 0,
srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy); cSize, srcSize, (int)zc->appliedParams.cParams.strategy);
if (action == ZSTD_RUST_TARGET_CBLOCK_ACTION_ERROR) {
FORWARD_IF_ERROR(cSize, "ZSTD_compressSuperBlock failed"); FORWARD_IF_ERROR(cSize, "ZSTD_compressSuperBlock failed");
if (cSize != 0 && cSize < maxCSize + ZSTD_blockHeaderSize) { }
if (action == ZSTD_RUST_TARGET_CBLOCK_ACTION_COMPRESSED) {
ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
return cSize; return cSize;
} }
} }
}
} /* if (bss == ZSTDbss_compress)*/ } /* if (bss == ZSTDbss_compress)*/
DEBUGLOG(6, "Resorting to ZSTD_noCompressBlock()"); DEBUGLOG(6, "Resorting to ZSTD_noCompressBlock()");
+173
View File
@@ -19,6 +19,7 @@ use crate::zstd_compress_api::ZSTD_compressBound;
use crate::zstd_compress_frame::{ use crate::zstd_compress_frame::{
write_raw_block, ZSTD_rust_writeFrameHeader, ZSTD_writeLastEmptyBlock, write_raw_block, ZSTD_rust_writeFrameHeader, ZSTD_writeLastEmptyBlock,
}; };
use crate::zstd_compress_literals::min_gain;
use crate::zstd_compress_params::{ use crate::zstd_compress_params::{
ZSTD_rust_params_adjustCParams, ZSTD_rust_params_maxNbSeq, ZSTD_rust_params_selectCParams, ZSTD_rust_params_adjustCParams, ZSTD_rust_params_maxNbSeq, ZSTD_rust_params_selectCParams,
ZSTD_RUST_CPM_NO_ATTACH_DICT, ZSTD_RUST_PS_DISABLE, ZSTD_RUST_CPM_NO_ATTACH_DICT, ZSTD_RUST_PS_DISABLE,
@@ -62,6 +63,8 @@ const ZSTD_FAST: c_int = 1;
const ZSTD_DFAST: c_int = 2; const ZSTD_DFAST: c_int = 2;
const ZSTD_BLOCKSIZE_MAX: usize = 1 << 17; const ZSTD_BLOCKSIZE_MAX: usize = 1 << 17;
const ZSTD_CONTENTSIZE_UNKNOWN: u64 = u64::MAX; const ZSTD_CONTENTSIZE_UNKNOWN: u64 = u64::MAX;
const ZSTD_TARGET_CBLOCK_BSS_COMPRESS: c_int = 0;
const ZSTD_BLOCK_HEADER_SIZE: usize = 3;
const ZSTD_ROWSIZE: usize = 16; const ZSTD_ROWSIZE: usize = 16;
const ZSTD_WINDOW_START_INDEX: u32 = 2; const ZSTD_WINDOW_START_INDEX: u32 = 2;
const ZSTD_DUBT_UNSORTED_MARK: u32 = 1; const ZSTD_DUBT_UNSORTED_MARK: u32 = 1;
@@ -75,6 +78,71 @@ const ZSTD_CHUNKSIZE_MAX: usize = u32::MAX as usize - ZSTD_CURRENT_MAX;
#[cfg(not(test))] #[cfg(not(test))]
const ZSTD_E_END: c_int = 2; const ZSTD_E_END: c_int = 2;
#[repr(i32)]
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
enum TargetCBlockAction {
Raw = 0,
Rle = 1,
Compressed = 2,
Error = 3,
}
#[inline]
fn target_c_block_size_action(
bss: c_int,
is_first_block: c_int,
maybe_rle: c_int,
is_rle: c_int,
c_size: usize,
src_size: usize,
strategy: c_int,
) -> TargetCBlockAction {
if bss != ZSTD_TARGET_CBLOCK_BSS_COMPRESS {
return TargetCBlockAction::Raw;
}
if is_first_block == 0 && maybe_rle != 0 && is_rle != 0 {
return TargetCBlockAction::Rle;
}
if c_size == 0 || c_size == ERROR(ZstdErrorCode::DstSizeTooSmall) {
return TargetCBlockAction::Raw;
}
if ERR_isError(c_size) {
return TargetCBlockAction::Error;
}
let max_c_size = src_size.wrapping_sub(min_gain(src_size, strategy));
if c_size < max_c_size.wrapping_add(ZSTD_BLOCK_HEADER_SIZE) {
TargetCBlockAction::Compressed
} else {
TargetCBlockAction::Raw
}
}
/// Classify the target-sized block policy without crossing the C context ABI.
///
/// C calls this once before the superblock attempt to classify the RLE
/// precondition, then again with the superblock result and zero RLE flags.
#[no_mangle]
pub extern "C" fn ZSTD_rust_targetCBlockSizeAction(
bss: c_int,
is_first_block: c_int,
maybe_rle: c_int,
is_rle: c_int,
c_size: usize,
src_size: usize,
strategy: c_int,
) -> c_int {
target_c_block_size_action(
bss,
is_first_block,
maybe_rle,
is_rle,
c_size,
src_size,
strategy,
) as c_int
}
#[cfg(not(test))] #[cfg(not(test))]
#[repr(C)] #[repr(C)]
pub struct ZSTD_inBuffer { pub struct ZSTD_inBuffer {
@@ -752,6 +820,111 @@ mod tests {
assert_eq!(ZSTD_rust_dictTooBig(ZSTD_CHUNKSIZE_MAX + 1), 1); assert_eq!(ZSTD_rust_dictTooBig(ZSTD_CHUNKSIZE_MAX + 1), 1);
} }
#[test]
fn target_block_policy_requires_compression_and_nonfirst_rle() {
assert_eq!(
target_c_block_size_action(0, 0, 1, 1, 0, 128, ZSTD_FAST),
TargetCBlockAction::Rle
);
assert_eq!(
target_c_block_size_action(0, 1, 1, 1, 0, 128, ZSTD_FAST),
TargetCBlockAction::Raw
);
assert_eq!(
target_c_block_size_action(0, 0, 0, 1, 0, 128, ZSTD_FAST),
TargetCBlockAction::Raw
);
assert_eq!(
target_c_block_size_action(0, 0, 1, 0, 0, 128, ZSTD_FAST),
TargetCBlockAction::Raw
);
assert_eq!(
ZSTD_rust_targetCBlockSizeAction(0, 0, 1, 1, 0, 128, ZSTD_FAST),
TargetCBlockAction::Rle as c_int
);
}
#[test]
fn target_block_policy_falls_back_for_uncompressed_and_small_results() {
assert_eq!(
target_c_block_size_action(1, 0, 0, 0, 0, 128, ZSTD_FAST),
TargetCBlockAction::Raw
);
assert_eq!(
target_c_block_size_action(
ZSTD_TARGET_CBLOCK_BSS_COMPRESS,
0,
0,
0,
ERROR(ZstdErrorCode::DstSizeTooSmall),
128,
ZSTD_FAST,
),
TargetCBlockAction::Raw
);
assert_eq!(
target_c_block_size_action(
ZSTD_TARGET_CBLOCK_BSS_COMPRESS,
0,
0,
0,
ERROR(ZstdErrorCode::Generic),
128,
ZSTD_FAST,
),
TargetCBlockAction::Error
);
assert_eq!(
target_c_block_size_action(ZSTD_TARGET_CBLOCK_BSS_COMPRESS, 0, 0, 0, 0, 128, ZSTD_FAST,),
TargetCBlockAction::Raw
);
}
#[test]
fn target_block_policy_uses_a_strict_three_byte_header_boundary() {
let src_size = 128;
let max_c_size = src_size - min_gain(src_size, ZSTD_FAST);
let compressed = max_c_size + ZSTD_BLOCK_HEADER_SIZE - 1;
let raw = max_c_size + ZSTD_BLOCK_HEADER_SIZE;
assert_eq!(
target_c_block_size_action(
ZSTD_TARGET_CBLOCK_BSS_COMPRESS,
0,
0,
0,
compressed,
src_size,
ZSTD_FAST,
),
TargetCBlockAction::Compressed
);
assert_eq!(
target_c_block_size_action(
ZSTD_TARGET_CBLOCK_BSS_COMPRESS,
0,
0,
0,
raw,
src_size,
ZSTD_FAST,
),
TargetCBlockAction::Raw
);
assert_eq!(
ZSTD_rust_targetCBlockSizeAction(
ZSTD_TARGET_CBLOCK_BSS_COMPRESS,
0,
0,
0,
compressed,
src_size,
ZSTD_FAST,
),
TargetCBlockAction::Compressed as c_int
);
}
#[test] #[test]
fn frame_progress_unknown_pledge_updates_counters() { fn frame_progress_unknown_pledge_updates_counters() {
let mut consumed = 7; let mut consumed = 7;