fix(legacy): preserve v0.7 FSE terminal symbols
The frozen v0.7 decoder's two-state FSE tail must emit state 1, reload, state 2, reload, and possibly a final state 1 symbol. The Rust port used current-state and end-of-stream early exits, so historical Huffman headers could stop one terminal weight short and corrupt a literal byte. The legacy corpus exposed this as the UTF-8 apostrophe's first byte changing from e2 to e0. Match the historical reload-driven tail and retain the exact v0.7 frame as a regression test. Test Plan: - `cargo test --manifest-path rust/Cargo.toml --no-default-features --features decompression,legacy-v07 legacy::zstd_v07::tests` -- passed (6 tests) - `cargo clippy --manifest-path rust/Cargo.toml --no-default-features --features decompression,legacy-v07 --lib -- -D warnings` -- passed - `rustfmt +nightly --edition 2021 rust/src/legacy/zstd_v07.rs` -- passed - `make -C tests -j2 legacy` -- blocked by the active dict-builder work's unrelated full-feature compilation gap
This commit is contained in:
+63
-16
@@ -614,32 +614,39 @@ unsafe fn fse_decompress_using_dtable(
|
||||
op = op.add(4);
|
||||
}
|
||||
|
||||
/* The frozen v0.7 decoder always emits the state-1 symbol first in the
|
||||
* tail. The reload after that symbol decides whether the final state-2
|
||||
* symbol is still available; a second reload can likewise leave one
|
||||
* final state-1 symbol to emit. In particular, do not use the current
|
||||
* state value as an early-stop condition: historical v0.7 streams rely
|
||||
* on these terminal symbols when reconstructing Huffman weights. */
|
||||
loop {
|
||||
if reload_dstream(&mut stream) > DSTREAM_COMPLETED
|
||||
|| op as usize == end_addr
|
||||
|| (end_of_dstream(&stream) && (fast || state1.state == 0))
|
||||
{
|
||||
break;
|
||||
if op as usize > end_addr.wrapping_sub(2) {
|
||||
return ERROR(ZstdErrorCode::DstSizeTooSmall);
|
||||
}
|
||||
*op = fse_decode_symbol(&mut state1, &mut stream, fast);
|
||||
op = op.add(1);
|
||||
if reload_dstream(&mut stream) > DSTREAM_COMPLETED
|
||||
|| op as usize == end_addr
|
||||
|| (end_of_dstream(&stream) && (fast || state2.state == 0))
|
||||
{
|
||||
|
||||
if reload_dstream(&mut stream) == DSTREAM_TOO_FAR {
|
||||
*op = fse_decode_symbol(&mut state2, &mut stream, fast);
|
||||
op = op.add(1);
|
||||
break;
|
||||
}
|
||||
|
||||
if op as usize > end_addr.wrapping_sub(2) {
|
||||
return ERROR(ZstdErrorCode::DstSizeTooSmall);
|
||||
}
|
||||
*op = fse_decode_symbol(&mut state2, &mut stream, fast);
|
||||
op = op.add(1);
|
||||
|
||||
if reload_dstream(&mut stream) == DSTREAM_TOO_FAR {
|
||||
*op = fse_decode_symbol(&mut state1, &mut stream, fast);
|
||||
op = op.add(1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if end_of_dstream(&stream) && state1.state == 0 && state2.state == 0 {
|
||||
return (op as usize) - (start as usize);
|
||||
}
|
||||
if op as usize == end_addr {
|
||||
return ERROR(ZstdErrorCode::DstSizeTooSmall);
|
||||
}
|
||||
ERROR(ZstdErrorCode::CorruptionDetected)
|
||||
(op as usize) - (start as usize)
|
||||
}
|
||||
|
||||
unsafe fn fse_decompress(
|
||||
@@ -3926,6 +3933,46 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn historical_v07_huffman_frame_preserves_terminal_fse_symbols() {
|
||||
let frame: [u8; 178] = [
|
||||
0x27, 0xb5, 0x2f, 0xfd, 0x20, 0xef, 0x00, 0x00, 0xa6, 0x12, 0xe4, 0x84, 0x1f, 0xb0,
|
||||
0x01, 0x10, 0x00, 0x00, 0x00, 0x35, 0x59, 0xa6, 0xe7, 0xa1, 0xef, 0x7c, 0xfc, 0xbd,
|
||||
0x3f, 0xff, 0x9f, 0xef, 0xee, 0xef, 0x61, 0xc3, 0xaa, 0x31, 0x1d, 0x34, 0x38, 0x22,
|
||||
0x22, 0x04, 0x44, 0x21, 0x80, 0x32, 0xad, 0x28, 0xf3, 0xd6, 0x28, 0x0c, 0x0a, 0x0e,
|
||||
0xd6, 0x5c, 0xac, 0x19, 0x8d, 0x20, 0x5f, 0x45, 0x02, 0x2e, 0x17, 0x50, 0x66, 0x6d,
|
||||
0xac, 0x8b, 0x9c, 0x6e, 0x07, 0x73, 0x46, 0xbb, 0x44, 0x14, 0xe7, 0x98, 0xc3, 0xb9,
|
||||
0x17, 0x32, 0x6e, 0x33, 0x7c, 0x0e, 0x21, 0xb1, 0xdb, 0xcb, 0x89, 0x51, 0x23, 0x34,
|
||||
0xab, 0x9d, 0xbc, 0x6d, 0x20, 0xf5, 0x03, 0xa9, 0x91, 0x4c, 0x2e, 0x1f, 0x59, 0xdb,
|
||||
0xd9, 0x35, 0x67, 0x4b, 0x0c, 0x95, 0x79, 0x10, 0x00, 0x85, 0xa6, 0x96, 0x95, 0x2e,
|
||||
0xdf, 0x78, 0x7b, 0x4a, 0x5c, 0x09, 0x76, 0x97, 0xd1, 0x5c, 0x96, 0x12, 0x75, 0x35,
|
||||
0xa3, 0x55, 0x4a, 0xd4, 0x0b, 0x00, 0x35, 0x0b, 0x71, 0xb5, 0xc0, 0x2a, 0x5c, 0xe6,
|
||||
0x08, 0x45, 0xf1, 0x39, 0x43, 0xf1, 0x1c, 0x4b, 0x54, 0x10, 0x9d, 0x31, 0x50, 0x85,
|
||||
0x4b, 0x54, 0x0e, 0x01, 0x4b, 0x3d, 0x01, 0xc0, 0x00, 0x00,
|
||||
];
|
||||
let mut expected = Vec::new();
|
||||
expected.extend_from_slice(
|
||||
b"snowden is snowed in / he's now then in his snow den / when does the snow end?\n",
|
||||
);
|
||||
expected.extend_from_slice(
|
||||
b"goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n",
|
||||
);
|
||||
expected.extend_from_slice(
|
||||
b"when life shuts a door, / just open it. it\xe2\x80\x99s a door. / that is how doors work.\n",
|
||||
);
|
||||
let mut output = vec![0u8; expected.len()];
|
||||
unsafe {
|
||||
let decoded = ZSTDv07_decompress(
|
||||
output.as_mut_ptr().cast(),
|
||||
output.len(),
|
||||
frame.as_ptr().cast(),
|
||||
frame.len(),
|
||||
);
|
||||
assert_eq!(decoded, expected.len());
|
||||
}
|
||||
assert_eq!(output, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skippable_stream_consumes_payload_without_output() {
|
||||
let frame: [u8; 11] = [0x50, 0x2a, 0x4d, 0x18, 3, 0, 0, 0, 1, 2, 3];
|
||||
|
||||
Reference in New Issue
Block a user