diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index bbe0e6d5f..84b674974 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -266,6 +266,8 @@ void ZSTD_rust_storeLastLiterals(SeqStore_t* seqStorePtr, void ZSTD_rust_resetSeqStore(SeqStore_t* ssPtr); size_t ZSTD_rust_fastSequenceLengthSum(const ZSTD_Sequence* seqBuf, size_t seqBufSize); +size_t ZSTD_rust_convertSequencesNoRepcodes( + SeqDef* dstSeqs, const ZSTD_Sequence* inSeqs, size_t nbSequences); int ZSTD_rust_isRLE(const BYTE* src, size_t length); int ZSTD_rust_maybeRLE(const SeqStore_t* seqStore); size_t ZSTD_rust_postProcessSequenceProducerResult( @@ -5748,205 +5750,16 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* cctx, return cSize; } - -#if defined(__AVX2__) - -#include /* AVX2 intrinsics */ - -/* - * Convert 2 sequences per iteration, using AVX2 intrinsics: - * - offset -> offBase = offset + 2 - * - litLength -> (U16) litLength - * - matchLength -> (U16)(matchLength - 3) - * - rep is ignored - * Store only 8 bytes per SeqDef (offBase[4], litLength[2], mlBase[2]). - * - * At the end, instead of extracting two __m128i, - * we use _mm256_permute4x64_epi64(..., 0xE8) to move lane2 into lane1, - * then store the lower 16 bytes in one go. - * - * @returns 0 on succes, with no long length detected - * @returns > 0 if there is one long length (> 65535), - * indicating the position, and type. - */ +/* Sequence conversion is implemented in Rust; this wrapper keeps the + * existing C-side block orchestration and long-length bookkeeping intact. */ static size_t convertSequences_noRepcodes( SeqDef* dstSeqs, const ZSTD_Sequence* inSeqs, size_t nbSequences) { - /* - * addition: - * For each 128-bit half: (offset+2, litLength+0, matchLength-3, rep+0) - */ - const __m256i addition = _mm256_setr_epi32( - ZSTD_REP_NUM, 0, -MINMATCH, 0, /* for sequence i */ - ZSTD_REP_NUM, 0, -MINMATCH, 0 /* for sequence i+1 */ - ); - - /* limit: check if there is a long length */ - const __m256i limit = _mm256_set1_epi32(65535); - - /* - * shuffle mask for byte-level rearrangement in each 128-bit half: - * - * Input layout (after addition) per 128-bit half: - * [ offset+2 (4 bytes) | litLength (4 bytes) | matchLength (4 bytes) | rep (4 bytes) ] - * We only need: - * offBase (4 bytes) = offset+2 - * litLength (2 bytes) = low 2 bytes of litLength - * mlBase (2 bytes) = low 2 bytes of (matchLength) - * => Bytes [0..3, 4..5, 8..9], zero the rest. - */ - const __m256i mask = _mm256_setr_epi8( - /* For the lower 128 bits => sequence i */ - 0, 1, 2, 3, /* offset+2 */ - 4, 5, /* litLength (16 bits) */ - 8, 9, /* matchLength (16 bits) */ - (BYTE)0x80, (BYTE)0x80, (BYTE)0x80, (BYTE)0x80, - (BYTE)0x80, (BYTE)0x80, (BYTE)0x80, (BYTE)0x80, - - /* For the upper 128 bits => sequence i+1 */ - 16,17,18,19, /* offset+2 */ - 20,21, /* litLength */ - 24,25, /* matchLength */ - (BYTE)0x80, (BYTE)0x80, (BYTE)0x80, (BYTE)0x80, - (BYTE)0x80, (BYTE)0x80, (BYTE)0x80, (BYTE)0x80 - ); - - /* - * Next, we'll use _mm256_permute4x64_epi64(vshf, 0xE8). - * Explanation of 0xE8 = 11101000b => [lane0, lane2, lane2, lane3]. - * So the lower 128 bits become [lane0, lane2] => combining seq0 and seq1. - */ -#define PERM_LANE_0X_E8 0xE8 /* [0,2,2,3] in lane indices */ - - size_t longLen = 0, i = 0; - - /* AVX permutation depends on the specific definition of target structures */ - ZSTD_STATIC_ASSERT(sizeof(ZSTD_Sequence) == 16); - ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, offset) == 0); - ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, litLength) == 4); - ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, matchLength) == 8); - ZSTD_STATIC_ASSERT(sizeof(SeqDef) == 8); - ZSTD_STATIC_ASSERT(offsetof(SeqDef, offBase) == 0); - ZSTD_STATIC_ASSERT(offsetof(SeqDef, litLength) == 4); - ZSTD_STATIC_ASSERT(offsetof(SeqDef, mlBase) == 6); - - /* Process 2 sequences per loop iteration */ - for (; i + 1 < nbSequences; i += 2) { - /* Load 2 ZSTD_Sequence (32 bytes) */ - __m256i vin = _mm256_loadu_si256((const __m256i*)(const void*)&inSeqs[i]); - - /* Add {2, 0, -3, 0} in each 128-bit half */ - __m256i vadd = _mm256_add_epi32(vin, addition); - - /* Check for long length */ - __m256i ll_cmp = _mm256_cmpgt_epi32(vadd, limit); /* 0xFFFFFFFF for element > 65535 */ - int ll_res = _mm256_movemask_epi8(ll_cmp); - - /* Shuffle bytes so each half gives us the 8 bytes we need */ - __m256i vshf = _mm256_shuffle_epi8(vadd, mask); - /* - * Now: - * Lane0 = seq0's 8 bytes - * Lane1 = 0 - * Lane2 = seq1's 8 bytes - * Lane3 = 0 - */ - - /* Permute 64-bit lanes => move Lane2 down into Lane1. */ - __m256i vperm = _mm256_permute4x64_epi64(vshf, PERM_LANE_0X_E8); - /* - * Now the lower 16 bytes (Lane0+Lane1) = [seq0, seq1]. - * The upper 16 bytes are [Lane2, Lane3] = [seq1, 0], but we won't use them. - */ - - /* Store only the lower 16 bytes => 2 SeqDef (8 bytes each) */ - _mm_storeu_si128((__m128i *)(void*)&dstSeqs[i], _mm256_castsi256_si128(vperm)); - /* - * This writes out 16 bytes total: - * - offset 0..7 => seq0 (offBase, litLength, mlBase) - * - offset 8..15 => seq1 (offBase, litLength, mlBase) - */ - - /* check (unlikely) long lengths > 65535 - * indices for lengths correspond to bits [4..7], [8..11], [20..23], [24..27] - * => combined mask = 0x0FF00FF0 - */ - if (UNLIKELY((ll_res & 0x0FF00FF0) != 0)) { - /* long length detected: let's figure out which one*/ - if (inSeqs[i].matchLength > 65535+MINMATCH) { - assert(longLen == 0); - longLen = i + 1; - } - if (inSeqs[i].litLength > 65535) { - assert(longLen == 0); - longLen = i + nbSequences + 1; - } - if (inSeqs[i+1].matchLength > 65535+MINMATCH) { - assert(longLen == 0); - longLen = i + 1 + 1; - } - if (inSeqs[i+1].litLength > 65535) { - assert(longLen == 0); - longLen = i + 1 + nbSequences + 1; - } - } - } - - /* Handle leftover if @nbSequences is odd */ - if (i < nbSequences) { - /* process last sequence */ - assert(i == nbSequences - 1); - dstSeqs[i].offBase = OFFSET_TO_OFFBASE(inSeqs[i].offset); - dstSeqs[i].litLength = (U16)inSeqs[i].litLength; - dstSeqs[i].mlBase = (U16)(inSeqs[i].matchLength - MINMATCH); - /* check (unlikely) long lengths > 65535 */ - if (UNLIKELY(inSeqs[i].matchLength > 65535+MINMATCH)) { - assert(longLen == 0); - longLen = i + 1; - } - if (UNLIKELY(inSeqs[i].litLength > 65535)) { - assert(longLen == 0); - longLen = i + nbSequences + 1; - } - } - - return longLen; + return ZSTD_rust_convertSequencesNoRepcodes(dstSeqs, inSeqs, nbSequences); } -/* the vector implementation could also be ported to SSSE3, - * but since this implementation is targeting modern systems (>= Sapphire Rapid), - * it's not useful to develop and maintain code for older pre-AVX2 platforms */ - -#else /* no AVX2 */ - -static size_t convertSequences_noRepcodes( - SeqDef* dstSeqs, - const ZSTD_Sequence* inSeqs, - size_t nbSequences) -{ - size_t longLen = 0; - size_t n; - for (n=0; n 65535 */ - if (UNLIKELY(inSeqs[n].matchLength > 65535+MINMATCH)) { - assert(longLen == 0); - longLen = n + 1; - } - if (UNLIKELY(inSeqs[n].litLength > 65535)) { - assert(longLen == 0); - longLen = n + nbSequences + 1; - } - } - return longLen; -} - -#endif - /* * Precondition: Sequences must end on an explicit Block Delimiter * @return: 0 on success, or an error code. diff --git a/rust/src/zstd_compress_stats.rs b/rust/src/zstd_compress_stats.rs index e9cd1f32d..a1036f068 100644 --- a/rust/src/zstd_compress_stats.rs +++ b/rust/src/zstd_compress_stats.rs @@ -158,6 +158,46 @@ pub struct ZSTD_Sequence { pub rep: u32, } +/// Converts public sequences to the internal no-repcodes `SeqDef` format. +/// +/// The return value is a side-band marker: zero means that every length fits +/// in its `u16` field; otherwise it is one plus the match-length sequence +/// index, or `nb_sequences + 1` plus the literal-length sequence index. This +/// is the Rust leaf for C's `convertSequences_noRepcodes()`. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_convertSequencesNoRepcodes( + dst_seqs: *mut SeqDef, + in_seqs: *const ZSTD_Sequence, + nb_sequences: usize, +) -> usize { + if nb_sequences == 0 { + return 0; + } + + let dst_seqs = unsafe { std::slice::from_raw_parts_mut(dst_seqs, nb_sequences) }; + let in_seqs = unsafe { std::slice::from_raw_parts(in_seqs, nb_sequences) }; + let mut long_length = 0usize; + + for (index, input) in in_seqs.iter().enumerate() { + dst_seqs[index] = SeqDef { + offBase: input.offset.wrapping_add(ZSTD_REP_NUM as u32), + litLength: input.litLength as u16, + mlBase: input.matchLength.wrapping_sub(MINMATCH as u32) as u16, + }; + + if input.matchLength > 65535 + MINMATCH as u32 { + debug_assert_eq!(long_length, 0); + long_length = index + 1; + } + if input.litLength > 65535 { + debug_assert_eq!(long_length, 0); + long_length = index + nb_sequences + 1; + } + } + + long_length +} + /// Converts a raw sequence offset to the stored offBase representation. /// /// This is the Rust leaf for C's `ZSTD_finalizeOffBase()`. The repcode @@ -1793,6 +1833,65 @@ mod tests { } } + #[test] + fn convert_sequences_no_repcodes_maps_fields_and_ignores_rep() { + let input = [ + ZSTD_Sequence { + offset: 1, + litLength: 7, + matchLength: 3, + rep: 99, + }, + ZSTD_Sequence { + offset: 1234, + litLength: 0x12345, + matchLength: 17, + rep: 1, + }, + ]; + let mut output = [SeqDef::default(); 2]; + + let long_length = unsafe { + ZSTD_rust_convertSequencesNoRepcodes(output.as_mut_ptr(), input.as_ptr(), input.len()) + }; + + assert_eq!(long_length, input.len() + 1 + 1); + assert_eq!(output[0].offBase, 1 + ZSTD_REP_NUM as u32); + assert_eq!(output[0].litLength, 7); + assert_eq!(output[0].mlBase, 0); + assert_eq!(output[1].offBase, 1234 + ZSTD_REP_NUM as u32); + assert_eq!(output[1].litLength, 0x2345); + assert_eq!(output[1].mlBase, 14); + } + + #[test] + fn convert_sequences_no_repcodes_marks_a_long_match() { + let input = [ZSTD_Sequence { + offset: 8, + litLength: 2, + matchLength: 65535 + MINMATCH as u32 + 1, + rep: 0, + }]; + let mut output = [SeqDef::default()]; + + let long_length = unsafe { + ZSTD_rust_convertSequencesNoRepcodes(output.as_mut_ptr(), input.as_ptr(), input.len()) + }; + + assert_eq!(long_length, 1); + assert_eq!(output[0].offBase, 8 + ZSTD_REP_NUM as u32); + assert_eq!(output[0].litLength, 2); + assert_eq!(output[0].mlBase, 0); + } + + #[test] + fn convert_sequences_no_repcodes_accepts_an_empty_range() { + let result = unsafe { + ZSTD_rust_convertSequencesNoRepcodes(std::ptr::null_mut(), std::ptr::null(), 0) + }; + assert_eq!(result, 0); + } + #[test] fn compressed_block_state_reset_restores_repcodes_and_repeat_modes() { let mut block_state =