fix(legacy): preserve v0.7 FSE terminal symbols

The frozen v0.7 decoder's two-state FSE tail must emit state 1, reload,
state 2, reload, and possibly a final state 1 symbol.  The Rust port used
current-state and end-of-stream early exits, so historical Huffman headers
could stop one terminal weight short and corrupt a literal byte.  The legacy
corpus exposed this as the UTF-8 apostrophe's first byte changing from e2 to
e0.  Match the historical reload-driven tail and retain the exact v0.7 frame
as a regression test.

Test Plan:
- `cargo test --manifest-path rust/Cargo.toml --no-default-features --features decompression,legacy-v07 legacy::zstd_v07::tests` -- passed (6 tests)
- `cargo clippy --manifest-path rust/Cargo.toml --no-default-features --features decompression,legacy-v07 --lib -- -D warnings` -- passed
- `rustfmt +nightly --edition 2021 rust/src/legacy/zstd_v07.rs` -- passed
- `make -C tests -j2 legacy` -- blocked by the active dict-builder work's
  unrelated full-feature compilation gap
This commit is contained in:
2026-07-18 15:42:43 +02:00
parent 66cb254532
commit 821f01cf44
+63 -16
View File
@@ -614,32 +614,39 @@ unsafe fn fse_decompress_using_dtable(
op = op.add(4);
}
/* The frozen v0.7 decoder always emits the state-1 symbol first in the
* tail. The reload after that symbol decides whether the final state-2
* symbol is still available; a second reload can likewise leave one
* final state-1 symbol to emit. In particular, do not use the current
* state value as an early-stop condition: historical v0.7 streams rely
* on these terminal symbols when reconstructing Huffman weights. */
loop {
if reload_dstream(&mut stream) > DSTREAM_COMPLETED
|| op as usize == end_addr
|| (end_of_dstream(&stream) && (fast || state1.state == 0))
{
break;
if op as usize > end_addr.wrapping_sub(2) {
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
*op = fse_decode_symbol(&mut state1, &mut stream, fast);
op = op.add(1);
if reload_dstream(&mut stream) > DSTREAM_COMPLETED
|| op as usize == end_addr
|| (end_of_dstream(&stream) && (fast || state2.state == 0))
{
if reload_dstream(&mut stream) == DSTREAM_TOO_FAR {
*op = fse_decode_symbol(&mut state2, &mut stream, fast);
op = op.add(1);
break;
}
if op as usize > end_addr.wrapping_sub(2) {
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
*op = fse_decode_symbol(&mut state2, &mut stream, fast);
op = op.add(1);
if reload_dstream(&mut stream) == DSTREAM_TOO_FAR {
*op = fse_decode_symbol(&mut state1, &mut stream, fast);
op = op.add(1);
break;
}
}
if end_of_dstream(&stream) && state1.state == 0 && state2.state == 0 {
return (op as usize) - (start as usize);
}
if op as usize == end_addr {
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
ERROR(ZstdErrorCode::CorruptionDetected)
(op as usize) - (start as usize)
}
unsafe fn fse_decompress(
@@ -3926,6 +3933,46 @@ mod tests {
}
}
#[test]
fn historical_v07_huffman_frame_preserves_terminal_fse_symbols() {
let frame: [u8; 178] = [
0x27, 0xb5, 0x2f, 0xfd, 0x20, 0xef, 0x00, 0x00, 0xa6, 0x12, 0xe4, 0x84, 0x1f, 0xb0,
0x01, 0x10, 0x00, 0x00, 0x00, 0x35, 0x59, 0xa6, 0xe7, 0xa1, 0xef, 0x7c, 0xfc, 0xbd,
0x3f, 0xff, 0x9f, 0xef, 0xee, 0xef, 0x61, 0xc3, 0xaa, 0x31, 0x1d, 0x34, 0x38, 0x22,
0x22, 0x04, 0x44, 0x21, 0x80, 0x32, 0xad, 0x28, 0xf3, 0xd6, 0x28, 0x0c, 0x0a, 0x0e,
0xd6, 0x5c, 0xac, 0x19, 0x8d, 0x20, 0x5f, 0x45, 0x02, 0x2e, 0x17, 0x50, 0x66, 0x6d,
0xac, 0x8b, 0x9c, 0x6e, 0x07, 0x73, 0x46, 0xbb, 0x44, 0x14, 0xe7, 0x98, 0xc3, 0xb9,
0x17, 0x32, 0x6e, 0x33, 0x7c, 0x0e, 0x21, 0xb1, 0xdb, 0xcb, 0x89, 0x51, 0x23, 0x34,
0xab, 0x9d, 0xbc, 0x6d, 0x20, 0xf5, 0x03, 0xa9, 0x91, 0x4c, 0x2e, 0x1f, 0x59, 0xdb,
0xd9, 0x35, 0x67, 0x4b, 0x0c, 0x95, 0x79, 0x10, 0x00, 0x85, 0xa6, 0x96, 0x95, 0x2e,
0xdf, 0x78, 0x7b, 0x4a, 0x5c, 0x09, 0x76, 0x97, 0xd1, 0x5c, 0x96, 0x12, 0x75, 0x35,
0xa3, 0x55, 0x4a, 0xd4, 0x0b, 0x00, 0x35, 0x0b, 0x71, 0xb5, 0xc0, 0x2a, 0x5c, 0xe6,
0x08, 0x45, 0xf1, 0x39, 0x43, 0xf1, 0x1c, 0x4b, 0x54, 0x10, 0x9d, 0x31, 0x50, 0x85,
0x4b, 0x54, 0x0e, 0x01, 0x4b, 0x3d, 0x01, 0xc0, 0x00, 0x00,
];
let mut expected = Vec::new();
expected.extend_from_slice(
b"snowden is snowed in / he's now then in his snow den / when does the snow end?\n",
);
expected.extend_from_slice(
b"goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n",
);
expected.extend_from_slice(
b"when life shuts a door, / just open it. it\xe2\x80\x99s a door. / that is how doors work.\n",
);
let mut output = vec![0u8; expected.len()];
unsafe {
let decoded = ZSTDv07_decompress(
output.as_mut_ptr().cast(),
output.len(),
frame.as_ptr().cast(),
frame.len(),
);
assert_eq!(decoded, expected.len());
}
assert_eq!(output, expected);
}
#[test]
fn skippable_stream_consumes_payload_without_output() {
let frame: [u8; 11] = [0x50, 0x2a, 0x4d, 0x18, 3, 0, 0, 0, 1, 2, 3];