feat(compress): move ordinary block emission into Rust
Move the post-sequence-store body of ZSTD_compressBlock_internal behind a small C/Rust state projection. C continues to build the sequence store and handle the no-compress and sequence-producer error paths, while Rust now owns sequence collection, entropy emission, the legacy non-first-frame RLE gate, compressed-block confirmation, and offcode repeat cleanup. Remove the C wrappers that became dead after those leaves moved behind the Rust body. The first integration run exposed that leaving the old C finalization label in place confirmed compressed block state twice, undoing Rust's pointer swap and breaking a later sparse-file checksum. The C wrapper now returns directly for the Rust-owned path and retains only its C-owned no-compress cleanup. Test Plan: - cargo test --manifest-path rust/Cargo.toml --lib -- --test-threads=1 - cargo check --manifest-path rust/Cargo.toml --lib - cargo clippy --manifest-path rust/Cargo.toml --lib -- -D warnings - cargo clippy --manifest-path rust/Cargo.toml -- -D warnings - cargo +nightly fmt --manifest-path rust/Cargo.toml -- --check - make -B -C lib -j2 lib - make -B -C tests -j2 test-zstd - make -B -C tests -j2 test-cli-tests - ZSTREAM_TESTTIME=-T2s make -B -C tests -j2 test-zstream - FUZZERTEST=-T5s make -B -C tests -j2 test-fuzzer - make -B -C tests/fuzz -j2 all - make -B -C tests/fuzz -j2 sequence_compression_api - cargo clippy --manifest-path rust/Cargo.toml --tests -- -D warnings (pre-existing manual_repeat_n failure) - cargo clippy --manifest-path rust/Cargo.toml --benches -- -D warnings (pre-existing manual_repeat_n failure)
This commit is contained in:
@@ -275,6 +275,52 @@ const _: () = {
|
||||
);
|
||||
};
|
||||
|
||||
/// Explicit projection of the state used by `ZSTD_compressBlock_internal`.
|
||||
///
|
||||
/// Sequence-store construction and the no-compress fallback remain in C.
|
||||
/// Rust owns sequence collection, entropy emission, the legacy first-block
|
||||
/// RLE gate, and the compressed-block state transitions after the store is
|
||||
/// ready.
|
||||
#[repr(C)]
|
||||
pub struct ZSTD_rust_blockInternalState {
|
||||
seq_store: *const SeqStore_t,
|
||||
prev_c_block: *mut *mut ZSTD_compressedBlockState_t,
|
||||
next_c_block: *mut *mut ZSTD_compressedBlockState_t,
|
||||
tmp_workspace: *mut c_void,
|
||||
tmp_wksp_size: usize,
|
||||
seq_collector: *mut SeqCollector,
|
||||
strategy: c_int,
|
||||
disable_literal_compression: c_int,
|
||||
bmi2: c_int,
|
||||
is_first_block: c_int,
|
||||
}
|
||||
|
||||
const _: () = {
|
||||
assert!(offset_of!(ZSTD_rust_blockInternalState, seq_store) == 0);
|
||||
assert!(offset_of!(ZSTD_rust_blockInternalState, prev_c_block) == size_of::<usize>());
|
||||
assert!(offset_of!(ZSTD_rust_blockInternalState, next_c_block) == 2 * size_of::<usize>());
|
||||
assert!(offset_of!(ZSTD_rust_blockInternalState, tmp_workspace) == 3 * size_of::<usize>());
|
||||
assert!(offset_of!(ZSTD_rust_blockInternalState, tmp_wksp_size) == 4 * size_of::<usize>());
|
||||
assert!(offset_of!(ZSTD_rust_blockInternalState, seq_collector) == 5 * size_of::<usize>());
|
||||
assert!(offset_of!(ZSTD_rust_blockInternalState, strategy) == 6 * size_of::<usize>());
|
||||
assert!(
|
||||
offset_of!(ZSTD_rust_blockInternalState, disable_literal_compression)
|
||||
== 6 * size_of::<usize>() + size_of::<c_int>()
|
||||
);
|
||||
assert!(
|
||||
offset_of!(ZSTD_rust_blockInternalState, bmi2)
|
||||
== 6 * size_of::<usize>() + 2 * size_of::<c_int>()
|
||||
);
|
||||
assert!(
|
||||
offset_of!(ZSTD_rust_blockInternalState, is_first_block)
|
||||
== 6 * size_of::<usize>() + 3 * size_of::<c_int>()
|
||||
);
|
||||
assert!(
|
||||
size_of::<ZSTD_rust_blockInternalState>()
|
||||
== 6 * size_of::<usize>() + 4 * size_of::<c_int>()
|
||||
);
|
||||
};
|
||||
|
||||
/// Explicit projection of the state used by the target-sized block body.
|
||||
///
|
||||
/// Sequence-store construction and the matchfinder remain in C. This state
|
||||
@@ -398,6 +444,120 @@ impl SingleBlockSeams {
|
||||
}
|
||||
}
|
||||
|
||||
/// Rust implementation of `ZSTD_compressBlock_internal` after the C caller
|
||||
/// has built the sequence store.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
unsafe fn compress_block_internal_body_with(
|
||||
state: &ZSTD_rust_blockInternalState,
|
||||
dst: *mut c_void,
|
||||
dst_capacity: usize,
|
||||
src: *const c_void,
|
||||
src_size: usize,
|
||||
frame: c_uint,
|
||||
seams: SingleBlockSeams,
|
||||
) -> usize {
|
||||
const RLE_MAX_LENGTH: usize = 25;
|
||||
const FSE_REPEAT_CHECK: c_int = 1;
|
||||
const FSE_REPEAT_VALID: c_int = 2;
|
||||
|
||||
if state.seq_store.is_null()
|
||||
|| state.prev_c_block.is_null()
|
||||
|| state.next_c_block.is_null()
|
||||
|| state.seq_collector.is_null()
|
||||
{
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
}
|
||||
|
||||
let prev_c_block = unsafe { *state.prev_c_block };
|
||||
let next_c_block = unsafe { *state.next_c_block };
|
||||
if prev_c_block.is_null() || next_c_block.is_null() {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
}
|
||||
|
||||
if unsafe { (*state.seq_collector).collectSequences } != 0 {
|
||||
let result = unsafe {
|
||||
(seams.copy_sequences)(
|
||||
state.seq_collector,
|
||||
state.seq_store,
|
||||
(*prev_c_block).rep.as_ptr(),
|
||||
)
|
||||
};
|
||||
if ERR_isError(result) {
|
||||
return result;
|
||||
}
|
||||
unsafe { (seams.confirm)(state.prev_c_block, state.next_c_block) };
|
||||
return 0;
|
||||
}
|
||||
|
||||
let mut c_size = unsafe {
|
||||
(seams.entropy_compress)(
|
||||
state.seq_store,
|
||||
ptr::addr_of!((*prev_c_block).entropy),
|
||||
ptr::addr_of_mut!((*next_c_block).entropy),
|
||||
state.strategy,
|
||||
state.disable_literal_compression,
|
||||
dst,
|
||||
dst_capacity,
|
||||
src_size,
|
||||
state.tmp_workspace,
|
||||
state.tmp_wksp_size,
|
||||
state.bmi2,
|
||||
)
|
||||
};
|
||||
|
||||
if frame != 0
|
||||
&& state.is_first_block == 0
|
||||
&& c_size < RLE_MAX_LENGTH
|
||||
&& unsafe { (seams.is_rle)(src.cast(), src_size) } != 0
|
||||
{
|
||||
/* Preserve the legacy decoder compatibility rule for the first frame
|
||||
* block: later repeated blocks may use the one-byte RLE form. */
|
||||
c_size = 1;
|
||||
unsafe {
|
||||
*dst.cast::<u8>() = *src.cast::<u8>();
|
||||
}
|
||||
}
|
||||
|
||||
if !ERR_isError(c_size) && c_size > 1 {
|
||||
unsafe { (seams.confirm)(state.prev_c_block, state.next_c_block) };
|
||||
}
|
||||
|
||||
let prev_c_block = unsafe { *state.prev_c_block };
|
||||
if prev_c_block.is_null() {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
}
|
||||
if unsafe { (*prev_c_block).entropy.fse.offcode_repeatMode } == FSE_REPEAT_VALID {
|
||||
unsafe { (*prev_c_block).entropy.fse.offcode_repeatMode = FSE_REPEAT_CHECK };
|
||||
}
|
||||
|
||||
c_size
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_rust_compressBlockInternal(
|
||||
state: *const ZSTD_rust_blockInternalState,
|
||||
dst: *mut c_void,
|
||||
dst_capacity: usize,
|
||||
src: *const c_void,
|
||||
src_size: usize,
|
||||
frame: c_uint,
|
||||
) -> usize {
|
||||
if state.is_null() {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
}
|
||||
unsafe {
|
||||
compress_block_internal_body_with(
|
||||
&*state,
|
||||
dst,
|
||||
dst_capacity,
|
||||
src,
|
||||
src_size,
|
||||
frame,
|
||||
SingleBlockSeams::production(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn sequence_block_action(
|
||||
is_first_block: c_int,
|
||||
@@ -3657,6 +3817,32 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn block_internal_test_state(
|
||||
seq_store: &mut SeqStore_t,
|
||||
prev_block: &mut ZSTD_compressedBlockState_t,
|
||||
next_block: &mut ZSTD_compressedBlockState_t,
|
||||
prev_c_block: &mut *mut ZSTD_compressedBlockState_t,
|
||||
next_c_block: &mut *mut ZSTD_compressedBlockState_t,
|
||||
seq_collector: &mut SeqCollector,
|
||||
strategy: c_int,
|
||||
is_first_block: c_int,
|
||||
) -> ZSTD_rust_blockInternalState {
|
||||
*prev_c_block = prev_block;
|
||||
*next_c_block = next_block;
|
||||
ZSTD_rust_blockInternalState {
|
||||
seq_store,
|
||||
prev_c_block,
|
||||
next_c_block,
|
||||
tmp_workspace: ptr::null_mut(),
|
||||
tmp_wksp_size: 0,
|
||||
seq_collector,
|
||||
strategy,
|
||||
disable_literal_compression: 0,
|
||||
bmi2: 0,
|
||||
is_first_block,
|
||||
}
|
||||
}
|
||||
|
||||
struct SplitBlockTestStore {
|
||||
seq_store: SeqStore_t,
|
||||
_sequences: Box<[SeqDef; 2]>,
|
||||
@@ -3789,6 +3975,147 @@ mod tests {
|
||||
assert!(std::ptr::eq(prev_c_block, &prev_block));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn block_internal_body_collects_sequences_before_entropy_emission() {
|
||||
let (mut seq_store, _sequences, _literals) = target_block_test_seq_store();
|
||||
let mut prev_block = zeroed_state();
|
||||
prev_block.entropy.fse.offcode_repeatMode = 2;
|
||||
let mut next_block = zeroed_state();
|
||||
let mut prev_c_block = ptr::null_mut();
|
||||
let mut next_c_block = ptr::null_mut();
|
||||
let mut output_sequences = [ZSTD_Sequence {
|
||||
offset: 0xa5,
|
||||
litLength: 0xa5,
|
||||
matchLength: 0xa5,
|
||||
rep: 0xa5,
|
||||
}];
|
||||
let mut seq_collector = SeqCollector {
|
||||
collectSequences: 1,
|
||||
seqStart: output_sequences.as_mut_ptr(),
|
||||
seqIndex: 0,
|
||||
maxSequences: output_sequences.len(),
|
||||
};
|
||||
let state = block_internal_test_state(
|
||||
&mut seq_store,
|
||||
&mut prev_block,
|
||||
&mut next_block,
|
||||
&mut prev_c_block,
|
||||
&mut next_c_block,
|
||||
&mut seq_collector,
|
||||
1,
|
||||
0,
|
||||
);
|
||||
let source = [0x5au8; 16];
|
||||
let mut output = [0xa5u8; 16];
|
||||
|
||||
let result = unsafe {
|
||||
compress_block_internal_body_with(
|
||||
&state,
|
||||
output.as_mut_ptr().cast(),
|
||||
output.len(),
|
||||
source.as_ptr().cast(),
|
||||
source.len(),
|
||||
1,
|
||||
single_block_test_seams(),
|
||||
)
|
||||
};
|
||||
|
||||
assert_eq!(result, 0);
|
||||
assert_eq!(seq_collector.seqIndex, 1);
|
||||
assert_eq!(output_sequences[0].litLength, 0);
|
||||
assert!(std::ptr::eq(prev_c_block, &next_block));
|
||||
assert!(std::ptr::eq(next_c_block, &prev_block));
|
||||
assert_eq!(prev_block.entropy.fse.offcode_repeatMode, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn block_internal_body_gates_rle_and_cleans_repeat_mode() {
|
||||
let (mut seq_store, _sequences, _literals) = target_block_test_seq_store();
|
||||
let mut prev_block = zeroed_state();
|
||||
prev_block.entropy.fse.offcode_repeatMode = 2;
|
||||
let mut next_block = zeroed_state();
|
||||
let mut prev_c_block = ptr::null_mut();
|
||||
let mut next_c_block = ptr::null_mut();
|
||||
let mut seq_collector = SeqCollector {
|
||||
collectSequences: 0,
|
||||
seqStart: ptr::null_mut(),
|
||||
seqIndex: 0,
|
||||
maxSequences: 0,
|
||||
};
|
||||
let state = block_internal_test_state(
|
||||
&mut seq_store,
|
||||
&mut prev_block,
|
||||
&mut next_block,
|
||||
&mut prev_c_block,
|
||||
&mut next_c_block,
|
||||
&mut seq_collector,
|
||||
1,
|
||||
0,
|
||||
);
|
||||
let source = [0x5au8; 16];
|
||||
let mut output = [0xa5u8; 32];
|
||||
|
||||
let result = unsafe {
|
||||
compress_block_internal_body_with(
|
||||
&state,
|
||||
output.as_mut_ptr().cast(),
|
||||
output.len(),
|
||||
source.as_ptr().cast(),
|
||||
source.len(),
|
||||
1,
|
||||
single_block_test_seams(),
|
||||
)
|
||||
};
|
||||
|
||||
assert_eq!(result, 1);
|
||||
assert_eq!(output[0], source[0]);
|
||||
assert_eq!(prev_block.entropy.fse.offcode_repeatMode, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn block_internal_body_propagates_entropy_error_through_repeat_cleanup() {
|
||||
let (mut seq_store, _sequences, _literals) = target_block_test_seq_store();
|
||||
let mut prev_block = zeroed_state();
|
||||
prev_block.entropy.fse.offcode_repeatMode = 2;
|
||||
let mut next_block = zeroed_state();
|
||||
let mut prev_c_block = ptr::null_mut();
|
||||
let mut next_c_block = ptr::null_mut();
|
||||
let mut seq_collector = SeqCollector {
|
||||
collectSequences: 0,
|
||||
seqStart: ptr::null_mut(),
|
||||
seqIndex: 0,
|
||||
maxSequences: 0,
|
||||
};
|
||||
let state = block_internal_test_state(
|
||||
&mut seq_store,
|
||||
&mut prev_block,
|
||||
&mut next_block,
|
||||
&mut prev_c_block,
|
||||
&mut next_c_block,
|
||||
&mut seq_collector,
|
||||
-1,
|
||||
0,
|
||||
);
|
||||
let source = [0x5au8; 16];
|
||||
let mut output = [0xa5u8; 32];
|
||||
|
||||
let result = unsafe {
|
||||
compress_block_internal_body_with(
|
||||
&state,
|
||||
output.as_mut_ptr().cast(),
|
||||
output.len(),
|
||||
source.as_ptr().cast(),
|
||||
source.len(),
|
||||
1,
|
||||
single_block_test_seams(),
|
||||
)
|
||||
};
|
||||
|
||||
assert_eq!(result, ERROR(ZstdErrorCode::Generic));
|
||||
assert_eq!(output, [0xa5u8; 32]);
|
||||
assert_eq!(prev_block.entropy.fse.offcode_repeatMode, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_block_body_propagates_entropy_errors_without_serializing() {
|
||||
let (seq_store, _sequences, _literals) = target_block_test_seq_store();
|
||||
|
||||
Reference in New Issue
Block a user