ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move external sequence conversion to Rust

Browse Source 
The external-sequence block path still converted public ZSTD_Sequence values
in C, with separate scalar and AVX2 implementations. Move that pure format
conversion into zstd_compress_stats.rs so both native configurations share one
implementation. Preserve offBase encoding, u16 truncation, the rep-ignored
contract, and the long-length side-band marker; leave C responsible for block
state, repcode resolution, and long-length bookkeeping.

Test Plan:
- `cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression` -- 225 passed
- `cargo clippy --manifest-path rust/Cargo.toml --no-default-features --features compression` plus benches/tests -- passed before and after formatting
- `cargo +nightly fmt --manifest-path rust/Cargo.toml --all` -- passed
- `make -B -C lib -j2 lib` -- passed
- `make -C tests -j2 test-zstream` -- passed, including 84 deterministic and 14,768 randomized cases
This commit is contained in:
ddidderr
2026-07-18 06:50:48 +02:00
parent 684963c1a9
commit 18f5419cc6
2 changed files with 104 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+5 -192
View File
@@ -266,6 +266,8 @@ void ZSTD_rust_storeLastLiterals(SeqStore_t* seqStorePtr,
void ZSTD_rust_resetSeqStore(SeqStore_t* ssPtr);
size_t ZSTD_rust_fastSequenceLengthSum(const ZSTD_Sequence* seqBuf,
size_t seqBufSize);
size_t ZSTD_rust_convertSequencesNoRepcodes(
SeqDef* dstSeqs, const ZSTD_Sequence* inSeqs, size_t nbSequences);
int ZSTD_rust_isRLE(const BYTE* src, size_t length);
int ZSTD_rust_maybeRLE(const SeqStore_t* seqStore);
size_t ZSTD_rust_postProcessSequenceProducerResult(
@@ -5748,205 +5750,16 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* cctx,
return cSize;
}
#if defined(__AVX2__)
#include <immintrin.h> /* AVX2 intrinsics */
/*
* Convert 2 sequences per iteration, using AVX2 intrinsics:
* - offset -> offBase = offset + 2
* - litLength -> (U16) litLength
* - matchLength -> (U16)(matchLength - 3)
* - rep is ignored
* Store only 8 bytes per SeqDef (offBase[4], litLength[2], mlBase[2]).
*
* At the end, instead of extracting two __m128i,
* we use _mm256_permute4x64_epi64(..., 0xE8) to move lane2 into lane1,
* then store the lower 16 bytes in one go.
*
* @returns 0 on succes, with no long length detected
* @returns > 0 if there is one long length (> 65535),
* indicating the position, and type.
*/
/* Sequence conversion is implemented in Rust; this wrapper keeps the
* existing C-side block orchestration and long-length bookkeeping intact. */
static size_t convertSequences_noRepcodes(
SeqDef* dstSeqs,
const ZSTD_Sequence* inSeqs,
size_t nbSequences)
{
/*
* addition:
* For each 128-bit half: (offset+2, litLength+0, matchLength-3, rep+0)
*/
const __m256i addition = _mm256_setr_epi32(
ZSTD_REP_NUM, 0, -MINMATCH, 0, /* for sequence i */
ZSTD_REP_NUM, 0, -MINMATCH, 0 /* for sequence i+1 */
);
/* limit: check if there is a long length */
const __m256i limit = _mm256_set1_epi32(65535);
/*
* shuffle mask for byte-level rearrangement in each 128-bit half:
*
* Input layout (after addition) per 128-bit half:
* [ offset+2 (4 bytes) | litLength (4 bytes) | matchLength (4 bytes) | rep (4 bytes) ]
* We only need:
* offBase (4 bytes) = offset+2
* litLength (2 bytes) = low 2 bytes of litLength
* mlBase (2 bytes) = low 2 bytes of (matchLength)
* => Bytes [0..3, 4..5, 8..9], zero the rest.
*/
const __m256i mask = _mm256_setr_epi8(
/* For the lower 128 bits => sequence i */
0, 1, 2, 3, /* offset+2 */
4, 5, /* litLength (16 bits) */
8, 9, /* matchLength (16 bits) */
(BYTE)0x80, (BYTE)0x80, (BYTE)0x80, (BYTE)0x80,
(BYTE)0x80, (BYTE)0x80, (BYTE)0x80, (BYTE)0x80,
/* For the upper 128 bits => sequence i+1 */
16,17,18,19, /* offset+2 */
20,21, /* litLength */
24,25, /* matchLength */
(BYTE)0x80, (BYTE)0x80, (BYTE)0x80, (BYTE)0x80,
(BYTE)0x80, (BYTE)0x80, (BYTE)0x80, (BYTE)0x80
);
/*
* Next, we'll use _mm256_permute4x64_epi64(vshf, 0xE8).
* Explanation of 0xE8 = 11101000b => [lane0, lane2, lane2, lane3].
* So the lower 128 bits become [lane0, lane2] => combining seq0 and seq1.
*/
#define PERM_LANE_0X_E8 0xE8 /* [0,2,2,3] in lane indices */
size_t longLen = 0, i = 0;
/* AVX permutation depends on the specific definition of target structures */
ZSTD_STATIC_ASSERT(sizeof(ZSTD_Sequence) == 16);
ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, offset) == 0);
ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, litLength) == 4);
ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, matchLength) == 8);
ZSTD_STATIC_ASSERT(sizeof(SeqDef) == 8);
ZSTD_STATIC_ASSERT(offsetof(SeqDef, offBase) == 0);
ZSTD_STATIC_ASSERT(offsetof(SeqDef, litLength) == 4);
ZSTD_STATIC_ASSERT(offsetof(SeqDef, mlBase) == 6);
/* Process 2 sequences per loop iteration */
for (; i + 1 < nbSequences; i += 2) {
/* Load 2 ZSTD_Sequence (32 bytes) */
__m256i vin = _mm256_loadu_si256((const __m256i*)(const void*)&inSeqs[i]);
/* Add {2, 0, -3, 0} in each 128-bit half */
__m256i vadd = _mm256_add_epi32(vin, addition);
/* Check for long length */
__m256i ll_cmp = _mm256_cmpgt_epi32(vadd, limit); /* 0xFFFFFFFF for element > 65535 */
int ll_res = _mm256_movemask_epi8(ll_cmp);
/* Shuffle bytes so each half gives us the 8 bytes we need */
__m256i vshf = _mm256_shuffle_epi8(vadd, mask);
/*
* Now:
* Lane0 = seq0's 8 bytes
* Lane1 = 0
* Lane2 = seq1's 8 bytes
* Lane3 = 0
*/
/* Permute 64-bit lanes => move Lane2 down into Lane1. */
__m256i vperm = _mm256_permute4x64_epi64(vshf, PERM_LANE_0X_E8);
/*
* Now the lower 16 bytes (Lane0+Lane1) = [seq0, seq1].
* The upper 16 bytes are [Lane2, Lane3] = [seq1, 0], but we won't use them.
*/
/* Store only the lower 16 bytes => 2 SeqDef (8 bytes each) */
_mm_storeu_si128((__m128i *)(void*)&dstSeqs[i], _mm256_castsi256_si128(vperm));
/*
* This writes out 16 bytes total:
* - offset 0..7 => seq0 (offBase, litLength, mlBase)
* - offset 8..15 => seq1 (offBase, litLength, mlBase)
*/
/* check (unlikely) long lengths > 65535
* indices for lengths correspond to bits [4..7], [8..11], [20..23], [24..27]
* => combined mask = 0x0FF00FF0
*/
if (UNLIKELY((ll_res & 0x0FF00FF0) != 0)) {
/* long length detected: let's figure out which one*/
if (inSeqs[i].matchLength > 65535+MINMATCH) {
assert(longLen == 0);
longLen = i + 1;
}
if (inSeqs[i].litLength > 65535) {
assert(longLen == 0);
longLen = i + nbSequences + 1;
}
if (inSeqs[i+1].matchLength > 65535+MINMATCH) {
assert(longLen == 0);
longLen = i + 1 + 1;
}
if (inSeqs[i+1].litLength > 65535) {
assert(longLen == 0);
longLen = i + 1 + nbSequences + 1;
}
}
}
/* Handle leftover if @nbSequences is odd */
if (i < nbSequences) {
/* process last sequence */
assert(i == nbSequences - 1);
dstSeqs[i].offBase = OFFSET_TO_OFFBASE(inSeqs[i].offset);
dstSeqs[i].litLength = (U16)inSeqs[i].litLength;
dstSeqs[i].mlBase = (U16)(inSeqs[i].matchLength - MINMATCH);
/* check (unlikely) long lengths > 65535 */
if (UNLIKELY(inSeqs[i].matchLength > 65535+MINMATCH)) {
assert(longLen == 0);
longLen = i + 1;
}
if (UNLIKELY(inSeqs[i].litLength > 65535)) {
assert(longLen == 0);
longLen = i + nbSequences + 1;
}
}
return longLen;
return ZSTD_rust_convertSequencesNoRepcodes(dstSeqs, inSeqs, nbSequences);
}
/* the vector implementation could also be ported to SSSE3,
* but since this implementation is targeting modern systems (>= Sapphire Rapid),
* it's not useful to develop and maintain code for older pre-AVX2 platforms */
#else /* no AVX2 */
static size_t convertSequences_noRepcodes(
SeqDef* dstSeqs,
const ZSTD_Sequence* inSeqs,
size_t nbSequences)
{
size_t longLen = 0;
size_t n;
for (n=0; n<nbSequences; n++) {
dstSeqs[n].offBase = OFFSET_TO_OFFBASE(inSeqs[n].offset);
dstSeqs[n].litLength = (U16)inSeqs[n].litLength;
dstSeqs[n].mlBase = (U16)(inSeqs[n].matchLength - MINMATCH);
/* check for long length > 65535 */
if (UNLIKELY(inSeqs[n].matchLength > 65535+MINMATCH)) {
assert(longLen == 0);
longLen = n + 1;
}
if (UNLIKELY(inSeqs[n].litLength > 65535)) {
assert(longLen == 0);
longLen = n + nbSequences + 1;
}
}
return longLen;
}
#endif
/*
* Precondition: Sequences must end on an explicit Block Delimiter
* @return: 0 on success, or an error code.
rust/src/zstd_compress_stats.rs
+99
View File
@@ -158,6 +158,46 @@ pub struct ZSTD_Sequence {
pub rep: u32,
}
/// Converts public sequences to the internal no-repcodes `SeqDef` format.
///
/// The return value is a side-band marker: zero means that every length fits
/// in its `u16` field; otherwise it is one plus the match-length sequence
/// index, or `nb_sequences + 1` plus the literal-length sequence index. This
/// is the Rust leaf for C's `convertSequences_noRepcodes()`.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_convertSequencesNoRepcodes(
dst_seqs: *mut SeqDef,
in_seqs: *const ZSTD_Sequence,
nb_sequences: usize,
) -> usize {
if nb_sequences == 0 {
return 0;
}
let dst_seqs = unsafe { std::slice::from_raw_parts_mut(dst_seqs, nb_sequences) };
let in_seqs = unsafe { std::slice::from_raw_parts(in_seqs, nb_sequences) };
let mut long_length = 0usize;
for (index, input) in in_seqs.iter().enumerate() {
dst_seqs[index] = SeqDef {
offBase: input.offset.wrapping_add(ZSTD_REP_NUM as u32),
litLength: input.litLength as u16,
mlBase: input.matchLength.wrapping_sub(MINMATCH as u32) as u16,
};
if input.matchLength > 65535 + MINMATCH as u32 {
debug_assert_eq!(long_length, 0);
long_length = index + 1;
}
if input.litLength > 65535 {
debug_assert_eq!(long_length, 0);
long_length = index + nb_sequences + 1;
}
}
long_length
}
/// Converts a raw sequence offset to the stored offBase representation.
///
/// This is the Rust leaf for C's `ZSTD_finalizeOffBase()`. The repcode
@@ -1793,6 +1833,65 @@ mod tests {
}
}
#[test]
fn convert_sequences_no_repcodes_maps_fields_and_ignores_rep() {
let input = [
ZSTD_Sequence {
offset: 1,
litLength: 7,
matchLength: 3,
rep: 99,
},
ZSTD_Sequence {
offset: 1234,
litLength: 0x12345,
matchLength: 17,
rep: 1,
},
];
let mut output = [SeqDef::default(); 2];
let long_length = unsafe {
ZSTD_rust_convertSequencesNoRepcodes(output.as_mut_ptr(), input.as_ptr(), input.len())
};
assert_eq!(long_length, input.len() + 1 + 1);
assert_eq!(output[0].offBase, 1 + ZSTD_REP_NUM as u32);
assert_eq!(output[0].litLength, 7);
assert_eq!(output[0].mlBase, 0);
assert_eq!(output[1].offBase, 1234 + ZSTD_REP_NUM as u32);
assert_eq!(output[1].litLength, 0x2345);
assert_eq!(output[1].mlBase, 14);
}
#[test]
fn convert_sequences_no_repcodes_marks_a_long_match() {
let input = [ZSTD_Sequence {
offset: 8,
litLength: 2,
matchLength: 65535 + MINMATCH as u32 + 1,
rep: 0,
}];
let mut output = [SeqDef::default()];
let long_length = unsafe {
ZSTD_rust_convertSequencesNoRepcodes(output.as_mut_ptr(), input.as_ptr(), input.len())
};
assert_eq!(long_length, 1);
assert_eq!(output[0].offBase, 8 + ZSTD_REP_NUM as u32);
assert_eq!(output[0].litLength, 2);
assert_eq!(output[0].mlBase, 0);
}
#[test]
fn convert_sequences_no_repcodes_accepts_an_empty_range() {
let result = unsafe {
ZSTD_rust_convertSequencesNoRepcodes(std::ptr::null_mut(), std::ptr::null(), 0)
};
assert_eq!(result, 0);
}
#[test]
fn compressed_block_state_reset_restores_repcodes_and_repeat_modes() {
let mut block_state =