From 80aaa8b24e892eeb1dc7e23169e4ccf41fc03ebe Mon Sep 17 00:00:00 2001 From: ddidderr Date: Sat, 18 Jul 2026 18:04:11 +0200 Subject: [PATCH] feat(legacy): complete the v0.3 Huffman decoder path The frozen v0.3 Rust decoder already handled the raw and X2 paths, but its historical Huffman X4 table construction and four-stream decode path were missing. Port the v0.3-specific X4 structures, rank calculations, table fill, stream selection, and decode behavior without reusing modern entropy code. Add ABI, malformed-frame, streaming-reset, and X4 regression coverage while keeping the existing declaration-only C boundary unchanged. Test Plan: - Focused v0.3 Rust tests -- 6 passed - Legacy-support-3 library build -- passed - `make -C tests -j2 test-legacy` -- passed - `make -C tests -j2 test-decodecorpus` -- passed - `cargo +nightly fmt --manifest-path rust/Cargo.toml -- --check` -- passed - `git diff --check` -- passed - Full clippy remains subject to the unrelated pre-existing zbuff dead-code baseline --- rust/src/legacy/zstd_v03.rs | 495 ++++++++++++++++++++++++++++++++++++ 1 file changed, 495 insertions(+) diff --git a/rust/src/legacy/zstd_v03.rs b/rust/src/legacy/zstd_v03.rs index a9564ee2c..be0bde75a 100644 --- a/rust/src/legacy/zstd_v03.rs +++ b/rust/src/legacy/zstd_v03.rs @@ -832,6 +832,388 @@ unsafe fn huf_decompress4x2_using_dtable( dst_size } +#[repr(C)] +#[derive(Clone, Copy, Default)] +struct HufDEltX4 { + sequence: u16, + nb_bits: u8, + length: u8, +} + +#[derive(Clone, Copy)] +struct SortedSymbol { + symbol: u8, + weight: u8, +} + +#[allow(clippy::too_many_arguments)] +unsafe fn huf_fill_dtable_x4_level2( + dtable: &mut [HufDEltX4], + size_log: u32, + consumed: u32, + rank_val_origin: &[u32; HUF_ABSOLUTE_MAX_TABLELOG + 1], + min_weight: i32, + sorted_symbols: &[SortedSymbol], + nb_bits_baseline: u32, + base_sequence: u16, +) { + let mut rank_val = *rank_val_origin; + + if min_weight > 1 { + let skip_size = rank_val[min_weight as usize] as usize; + let entry = HufDEltX4 { + sequence: base_sequence.to_le(), + nb_bits: consumed as u8, + length: 1, + }; + for cell in dtable.iter_mut().take(skip_size) { + *cell = entry; + } + } + + for sorted in sorted_symbols { + let symbol = sorted.symbol as u16; + let weight = sorted.weight as usize; + let nb_bits = nb_bits_baseline - sorted.weight as u32; + let length = 1usize << (size_log - nb_bits); + let start = rank_val[weight] as usize; + let end = start + length; + let entry = HufDEltX4 { + sequence: base_sequence.wrapping_add(symbol << 8).to_le(), + nb_bits: (nb_bits + consumed) as u8, + length: 2, + }; + for cell in &mut dtable[start..end] { + *cell = entry; + } + rank_val[weight] += length as u32; + } +} + +#[allow(clippy::needless_range_loop)] +#[allow(clippy::too_many_arguments)] +unsafe fn huf_fill_dtable_x4( + dtable: &mut [HufDEltX4], + target_log: u32, + sorted_list: &[SortedSymbol], + sorted_list_size: usize, + rank_start: &[u32; HUF_ABSOLUTE_MAX_TABLELOG + 1], + rank_val_origin: &[[u32; HUF_ABSOLUTE_MAX_TABLELOG + 1]; HUF_ABSOLUTE_MAX_TABLELOG], + max_weight: u32, + nb_bits_baseline: u32, +) { + let scale_log = nb_bits_baseline as i32 - target_log as i32; + let min_bits = nb_bits_baseline - max_weight; + let mut rank_val = rank_val_origin[0]; + + for sorted in sorted_list.iter().take(sorted_list_size) { + let weight = sorted.weight as usize; + let nb_bits = nb_bits_baseline - sorted.weight as u32; + let start = rank_val[weight] as usize; + let length = 1usize << (target_log - nb_bits); + + if target_log - nb_bits >= min_bits { + let min_weight = (nb_bits as i32 + scale_log).max(1); + let sorted_rank = rank_start[min_weight as usize] as usize; + huf_fill_dtable_x4_level2( + &mut dtable[start..start + length], + target_log - nb_bits, + nb_bits, + &rank_val_origin[nb_bits as usize], + min_weight, + &sorted_list[sorted_rank..], + nb_bits_baseline, + sorted.symbol as u16, + ); + } else { + let entry = HufDEltX4 { + sequence: (sorted.symbol as u16).to_le(), + nb_bits: nb_bits as u8, + length: 1, + }; + for cell in &mut dtable[start..start + length] { + *cell = entry; + } + } + rank_val[weight] += length as u32; + } +} + +#[allow(clippy::needless_range_loop)] +unsafe fn huf_read_dtable_x4( + dtable: &mut [HufDEltX4; 1 + (1 << HUF_MAX_TABLELOG)], + src: *const u8, + src_size: usize, +) -> usize { + let mem_log = dtable[0].sequence as u32; + if mem_log > HUF_ABSOLUTE_MAX_TABLELOG as u32 { + return ERROR(ZstdErrorCode::TableLogTooLarge); + } + + let mut weight_list = [0u8; HUF_MAX_SYMBOL_VALUE + 1]; + let mut sorted_symbols = [SortedSymbol { + symbol: 0, + weight: 0, + }; HUF_MAX_SYMBOL_VALUE + 1]; + let mut rank_stats = [0u32; HUF_ABSOLUTE_MAX_TABLELOG + 1]; + let mut nb_symbols = 0u32; + let mut table_log = 0u32; + let i_size = huf_read_stats( + &mut weight_list, + &mut rank_stats, + &mut nb_symbols, + &mut table_log, + src, + src_size, + ); + if ERR_isError(i_size) { + return i_size; + } + if table_log > mem_log { + return ERROR(ZstdErrorCode::TableLogTooLarge); + } + + let mut max_weight = table_log; + while rank_stats[max_weight as usize] == 0 { + if max_weight == 0 { + return ERROR(ZstdErrorCode::Generic); + } + max_weight -= 1; + } + + let mut rank_start = [0u32; HUF_ABSOLUTE_MAX_TABLELOG + 1]; + let mut next_rank_start = 0u32; + for weight in 1..=max_weight as usize { + let current = next_rank_start; + next_rank_start += rank_stats[weight]; + rank_start[weight] = current; + } + rank_start[0] = next_rank_start; + let size_of_sort = next_rank_start as usize; + + for (symbol, &weight) in weight_list.iter().enumerate().take(nb_symbols as usize) { + let rank = rank_start[weight as usize] as usize; + sorted_symbols[rank] = SortedSymbol { + symbol: symbol as u8, + weight, + }; + rank_start[weight as usize] += 1; + } + rank_start[0] = 0; + + let mut rank_val = [[0u32; HUF_ABSOLUTE_MAX_TABLELOG + 1]; HUF_ABSOLUTE_MAX_TABLELOG]; + let min_bits = table_log + 1 - max_weight; + let rescale = mem_log as i32 - table_log as i32 - 1; + let mut next_rank_val = 0u32; + for weight in 1..=max_weight as usize { + let current = next_rank_val; + let shift = (weight as i32 + rescale) as u32; + next_rank_val = next_rank_val.wrapping_add(rank_stats[weight] << shift); + rank_val[0][weight] = current; + } + for consumed in min_bits..=mem_log - min_bits { + for weight in 1..=max_weight as usize { + rank_val[consumed as usize][weight] = rank_val[0][weight] >> consumed; + } + } + + huf_fill_dtable_x4( + &mut dtable[1..], + mem_log, + &sorted_symbols, + size_of_sort, + &rank_start, + &rank_val, + max_weight, + table_log + 1, + ); + i_size +} + +#[inline] +unsafe fn huf_decode_symbol_x4( + op: *mut u8, + stream: &mut DStream, + table: *const HufDEltX4, + table_log: u32, +) -> u32 { + let entry = *table.add(look_bits_fast(stream, table_log)); + ptr::copy_nonoverlapping( + &entry.sequence as *const u16 as *const u8, + op, + entry.length as usize, + ); + skip_bits(stream, entry.nb_bits as u32); + entry.length as u32 +} + +#[inline] +unsafe fn huf_decode_symbol_x4_last( + op: *mut u8, + stream: &mut DStream, + table: *const HufDEltX4, + table_log: u32, +) -> u32 { + let entry = *table.add(look_bits_fast(stream, table_log)); + ptr::copy_nonoverlapping(&entry.sequence as *const u16 as *const u8, op, 1); + if entry.length == 1 { + skip_bits(stream, entry.nb_bits as u32); + } else if stream.bits_consumed < USIZE_BITS { + skip_bits(stream, entry.nb_bits as u32); + if stream.bits_consumed > USIZE_BITS { + stream.bits_consumed = USIZE_BITS; + } + } + 1 +} + +#[inline] +unsafe fn huf_decode_symbol_x4_0( + op: &mut *mut u8, + stream: &mut DStream, + table: *const HufDEltX4, + table_log: u32, +) { + *op = op.add(huf_decode_symbol_x4(*op, stream, table, table_log) as usize); +} + +#[inline] +unsafe fn huf_decode_symbol_x4_1( + op: &mut *mut u8, + stream: &mut DStream, + table: *const HufDEltX4, + table_log: u32, +) { + huf_decode_symbol_x4_0(op, stream, table, table_log); +} + +#[inline] +unsafe fn huf_decode_symbol_x4_2( + op: &mut *mut u8, + stream: &mut DStream, + table: *const HufDEltX4, + table_log: u32, +) { + if usize::BITS == 64 { + huf_decode_symbol_x4_0(op, stream, table, table_log); + } +} + +unsafe fn huf_decode_stream_x4( + dst: *mut u8, + dst_size: usize, + stream: &mut DStream, + table: *const HufDEltX4, + table_log: u32, +) -> usize { + let start = dst; + let end = dst.add(dst_size); + let mut op = dst; + + while reload_dstream(stream) == DSTREAM_UNFINISHED + && (end as usize).wrapping_sub(op as usize) > 7 + { + huf_decode_symbol_x4_2(&mut op, stream, table, table_log); + huf_decode_symbol_x4_1(&mut op, stream, table, table_log); + huf_decode_symbol_x4_2(&mut op, stream, table, table_log); + huf_decode_symbol_x4_0(&mut op, stream, table, table_log); + } + while reload_dstream(stream) == DSTREAM_UNFINISHED + && (end as usize).wrapping_sub(op as usize) >= 2 + { + huf_decode_symbol_x4_0(&mut op, stream, table, table_log); + } + while (end as usize).wrapping_sub(op as usize) >= 2 { + huf_decode_symbol_x4_0(&mut op, stream, table, table_log); + } + if op < end { + op = op.add(huf_decode_symbol_x4_last(op, stream, table, table_log) as usize); + } + (op as usize) - (start as usize) +} + +unsafe fn huf_decompress4x4_using_dtable( + dst: *mut u8, + dst_size: usize, + c_src: *const u8, + c_src_size: usize, + dtable: &[HufDEltX4; 1 + (1 << HUF_MAX_TABLELOG)], +) -> usize { + if c_src_size < 10 { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + let length1 = read_le16(c_src) as usize; + let length2 = read_le16(c_src.add(2)) as usize; + let length3 = read_le16(c_src.add(4)) as usize; + let payload = length1 + .checked_add(length2) + .and_then(|v| v.checked_add(length3)) + .and_then(|v| v.checked_add(6)); + let payload = match payload { + Some(value) if value <= c_src_size => value, + _ => return ERROR(ZstdErrorCode::CorruptionDetected), + }; + let length4 = c_src_size - payload; + let sources = [ + c_src.add(6), + c_src.add(6 + length1), + c_src.add(6 + length1 + length2), + c_src.add(6 + length1 + length2 + length3), + ]; + let lengths = [length1, length2, length3, length4]; + let segment = dst_size.div_ceil(4); + let starts = [ + dst, + dst.add(segment), + dst.add(segment * 2), + dst.add(segment * 3), + ]; + let sizes = [ + segment.min(dst_size), + segment.min(dst_size.saturating_sub(segment)), + segment.min(dst_size.saturating_sub(segment * 2)), + dst_size.saturating_sub(segment * 3), + ]; + let table = dtable.as_ptr().add(1); + let table_log = dtable[0].sequence as u32; + for index in 0..4 { + let mut stream = DStream { + bit_container: 0, + bits_consumed: 0, + ptr: ptr::null(), + start: ptr::null(), + }; + let error = init_dstream(&mut stream, sources[index], lengths[index]); + if ERR_isError(error) { + return error; + } + huf_decode_stream_x4(starts[index], sizes[index], &mut stream, table, table_log); + if !end_of_dstream(&stream) { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + } + dst_size +} + +const HUF_ALGO_TIME: [[(u32, u32); 3]; 16] = [ + [(0, 0), (1, 1), (2, 2)], + [(0, 0), (1, 1), (2, 2)], + [(38, 130), (1313, 74), (2151, 38)], + [(448, 128), (1353, 74), (2238, 41)], + [(556, 128), (1353, 74), (2238, 47)], + [(714, 128), (1418, 74), (2436, 53)], + [(883, 128), (1437, 74), (2464, 61)], + [(897, 128), (1515, 75), (2622, 68)], + [(926, 128), (1613, 75), (2730, 75)], + [(947, 128), (1729, 77), (3359, 77)], + [(1107, 128), (2083, 81), (4006, 84)], + [(1177, 128), (2379, 87), (4785, 88)], + [(1242, 128), (2415, 93), (5155, 84)], + [(1349, 128), (2644, 106), (5260, 106)], + [(1455, 128), (2422, 124), (4174, 124)], + [(722, 128), (1891, 145), (1936, 146)], +]; + unsafe fn huf_decompress( dst: *mut u8, dst_size: usize, @@ -852,6 +1234,33 @@ unsafe fn huf_decompress( ptr::write_bytes(dst, *c_src, dst_size); return dst_size; } + let q = (c_src_size.wrapping_mul(16) / dst_size).min(15); + let d256 = (dst_size >> 8) as u32; + let x2_time = HUF_ALGO_TIME[q][0] + .0 + .wrapping_add(HUF_ALGO_TIME[q][0].1.wrapping_mul(d256)); + let mut x4_time = HUF_ALGO_TIME[q][1] + .0 + .wrapping_add(HUF_ALGO_TIME[q][1].1.wrapping_mul(d256)); + x4_time = x4_time.wrapping_add(x4_time >> 4); + if x4_time < x2_time { + let mut dtable = [HufDEltX4::default(); 1 + (1 << HUF_MAX_TABLELOG)]; + dtable[0].sequence = HUF_MAX_TABLELOG as u16; + let header_size = huf_read_dtable_x4(&mut dtable, c_src, c_src_size); + if ERR_isError(header_size) { + return header_size; + } + if header_size >= c_src_size { + return ERROR(ZstdErrorCode::SrcSizeWrong); + } + return huf_decompress4x4_using_dtable( + dst, + dst_size, + c_src.add(header_size), + c_src_size - header_size, + &dtable, + ); + } let mut dtable = [0u16; 1 + (1 << HUF_MAX_TABLELOG)]; dtable[0] = HUF_MAX_TABLELOG as u16; let header_size = huf_read_dtable_x2(&mut dtable, c_src, c_src_size); @@ -1660,6 +2069,23 @@ mod tests { 0x7F, 0x00, 0xFC, 0x23, 0x10, 0xC0, 0x00, 0x00, ]; + const HUF_X4_FRAME: &[u8] = &[ + 0x23, 0xB5, 0x2F, 0xFD, 0x00, 0x00, 0xBB, 0xB0, 0x02, 0xC0, 0x10, 0x00, 0x1E, 0xB0, 0x01, + 0x02, 0x00, 0x00, 0x80, 0x00, 0xE8, 0x92, 0x34, 0x12, 0x97, 0xC8, 0xDF, 0xE9, 0xF3, 0xEF, + 0x53, 0xEA, 0x1D, 0x27, 0x4F, 0x0C, 0x44, 0x90, 0x0C, 0x8D, 0xF1, 0xB4, 0x89, 0x17, 0x00, + 0x18, 0x00, 0x18, 0x00, 0x3F, 0xE6, 0xE2, 0xE3, 0x74, 0xD6, 0xEC, 0xC9, 0x4A, 0xE0, 0x71, + 0x71, 0x42, 0x3E, 0x64, 0x4F, 0x6A, 0x45, 0x4E, 0x78, 0xEC, 0x49, 0x03, 0x3F, 0xC6, 0x80, + 0xAB, 0x8F, 0x75, 0x5E, 0x6F, 0x2E, 0x3E, 0x7E, 0xC6, 0xDC, 0x45, 0x69, 0x6C, 0xC5, 0xFD, + 0xC7, 0x40, 0xB8, 0x84, 0x8A, 0x01, 0xEB, 0xA8, 0xD1, 0x40, 0x39, 0x90, 0x4C, 0x64, 0xF8, + 0xEB, 0x53, 0xE6, 0x18, 0x0B, 0x67, 0x12, 0xAD, 0xB8, 0x99, 0xB3, 0x5A, 0x6F, 0x8A, 0x19, + 0x03, 0x01, 0x50, 0x67, 0x56, 0xF5, 0x9F, 0x35, 0x84, 0x60, 0xA0, 0x60, 0x91, 0xC9, 0x0A, + 0xDC, 0xAB, 0xAB, 0xE0, 0xE2, 0x81, 0xFA, 0xCF, 0xC6, 0xBA, 0x01, 0x0E, 0x00, 0x54, 0x00, + 0x00, 0x19, 0x00, 0x00, 0x54, 0x14, 0x00, 0x24, 0x24, 0x04, 0xFE, 0x04, 0x84, 0x4E, 0x41, + 0x00, 0x27, 0xE2, 0x02, 0xC4, 0xB1, 0x00, 0xD2, 0x51, 0x00, 0x79, 0x58, 0x41, 0x28, 0x00, + 0xE0, 0x0C, 0x01, 0x68, 0x65, 0x00, 0x04, 0x13, 0x0C, 0xDA, 0x0C, 0x80, 0x22, 0x06, 0xC0, + 0x00, 0x00, + ]; + unsafe fn decode(frame: &[u8], capacity: usize) -> Result, usize> { let mut output = vec![0u8; capacity]; let size = ZSTDv03_decompress( @@ -1689,6 +2115,17 @@ mod tests { } } + #[test] + fn decodes_huffman_x4_literals() { + let expected = + b"snowden is snowed in / he's now then in his snow den / when does the snow end?\n\ +goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n\ +when life shuts a door, / just open it. it\xE2\x80\x99s a door. / that is how doors work.\n"; + unsafe { + assert_eq!(decode(HUF_X4_FRAME, expected.len()).unwrap(), expected); + } + } + #[test] fn reports_frame_size_and_errors() { unsafe { @@ -1720,6 +2157,64 @@ mod tests { } } + #[test] + fn public_abi_preserves_error_and_context_behavior() { + assert_eq!(ZSTDv03_isError(0), 0); + assert_eq!(ZSTDv03_isError(ERROR(ZstdErrorCode::Generic)), 1); + assert_eq!(ZSTDv03_isError(usize::MAX), 1); + + unsafe { + let empty: [u8; 0] = []; + assert_eq!( + ZSTDv03_decompress( + ptr::null_mut(), + 0, + empty.as_ptr() as *const c_void, + empty.len(), + ), + ERROR(ZstdErrorCode::SrcSizeWrong) + ); + + let mut c_size = 123; + let mut bound = 456; + ZSTDv03_findFrameSizeInfoLegacy( + empty.as_ptr() as *const c_void, + empty.len(), + &mut c_size, + &mut bound, + ); + assert_eq!(c_size, ERROR(ZstdErrorCode::SrcSizeWrong)); + assert_eq!(bound, ZSTD_CONTENTSIZE_ERROR); + + let mut bad_magic = RAW_FRAME.to_vec(); + bad_magic[0] ^= 1; + ZSTDv03_findFrameSizeInfoLegacy( + bad_magic.as_ptr() as *const c_void, + bad_magic.len(), + &mut c_size, + &mut bound, + ); + assert_eq!(c_size, ERROR(ZstdErrorCode::PrefixUnknown)); + assert_eq!(bound, ZSTD_CONTENTSIZE_ERROR); + + let dctx = ZSTDv03_createDCtx(); + assert!(!dctx.is_null()); + let mut output = [0u8; 32]; + let decoded = ZSTDv03_decompressDCtx( + dctx as *mut c_void, + output.as_mut_ptr() as *mut c_void, + output.len(), + RAW_FRAME.as_ptr() as *const c_void, + RAW_FRAME.len(), + ); + assert_eq!(decoded, b"raw v0.3!!!".len()); + assert_eq!(&output[..decoded], b"raw v0.3!!!"); + assert_eq!(ZSTDv03_resetDCtx(dctx), 0); + assert_eq!(ZSTDv03_freeDCtx(dctx), 0); + assert_eq!(ZSTDv03_freeDCtx(ptr::null_mut()), 0); + } + } + #[test] fn streaming_raw_frame_matches_one_shot() { let expected = b"raw v0.3!!!";